2024-11-16 10:23:40,157 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-16 10:23:40,172 main DEBUG Took 0.012319 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-16 10:23:40,173 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-16 10:23:40,173 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-16 10:23:40,174 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-16 10:23:40,176 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 10:23:40,185 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-16 10:23:40,200 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 10:23:40,202 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 10:23:40,202 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 10:23:40,203 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 10:23:40,203 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 10:23:40,204 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 10:23:40,205 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 10:23:40,205 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 10:23:40,206 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 10:23:40,206 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 10:23:40,208 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 10:23:40,208 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 10:23:40,209 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 10:23:40,209 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 10:23:40,210 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 10:23:40,210 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 10:23:40,211 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 10:23:40,211 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 10:23:40,212 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 10:23:40,212 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 10:23:40,213 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 10:23:40,213 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 10:23:40,213 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 10:23:40,214 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 10:23:40,214 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 10:23:40,215 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-16 10:23:40,217 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 10:23:40,218 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-16 10:23:40,221 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-16 10:23:40,222 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-16 10:23:40,224 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-16 10:23:40,224 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-16 10:23:40,236 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-16 10:23:40,239 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-16 10:23:40,241 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-16 10:23:40,242 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-16 10:23:40,242 main DEBUG createAppenders(={Console}) 2024-11-16 10:23:40,243 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-16 10:23:40,244 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-16 10:23:40,244 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-16 10:23:40,245 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-16 10:23:40,245 main DEBUG OutputStream closed 2024-11-16 10:23:40,245 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-16 10:23:40,246 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-16 10:23:40,246 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-16 10:23:40,339 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-16 10:23:40,341 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-16 10:23:40,343 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-16 10:23:40,344 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-16 10:23:40,345 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-16 10:23:40,345 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-16 10:23:40,346 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-16 10:23:40,346 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-16 10:23:40,346 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-16 10:23:40,347 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-16 10:23:40,347 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-16 10:23:40,348 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-16 10:23:40,348 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-16 10:23:40,349 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-16 10:23:40,349 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-16 10:23:40,349 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-16 10:23:40,350 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-16 10:23:40,351 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-16 10:23:40,353 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16 10:23:40,354 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-16 10:23:40,354 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-16 10:23:40,355 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-16T10:23:40,655 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11 2024-11-16 10:23:40,659 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-16 10:23:40,659 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16T10:23:40,668 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-16T10:23:40,689 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-16T10:23:40,709 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/cluster_7ec35abe-b1d1-d9da-1787-50b9d114d89d, deleteOnExit=true 2024-11-16T10:23:40,710 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-16T10:23:40,712 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/test.cache.data in system properties and HBase conf 2024-11-16T10:23:40,713 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/hadoop.tmp.dir in system properties and HBase conf 2024-11-16T10:23:40,713 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/hadoop.log.dir in system properties and HBase conf 2024-11-16T10:23:40,715 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-16T10:23:40,716 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-16T10:23:40,716 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-16T10:23:40,825 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-16T10:23:40,939 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T10:23:40,944 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-16T10:23:40,945 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-16T10:23:40,946 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-16T10:23:40,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T10:23:40,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-16T10:23:40,948 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-16T10:23:40,949 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T10:23:40,949 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T10:23:40,950 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-16T10:23:40,951 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/nfs.dump.dir in system properties and HBase conf 2024-11-16T10:23:40,951 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/java.io.tmpdir in system properties and HBase conf 2024-11-16T10:23:40,952 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T10:23:40,952 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-16T10:23:40,953 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-16T10:23:42,168 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-16T10:23:42,277 INFO [Time-limited test {}] log.Log(170): Logging initialized @2937ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-16T10:23:42,381 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T10:23:42,471 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T10:23:42,505 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T10:23:42,506 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T10:23:42,508 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-16T10:23:42,526 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T10:23:42,530 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/hadoop.log.dir/,AVAILABLE} 2024-11-16T10:23:42,531 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T10:23:42,788 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/java.io.tmpdir/jetty-localhost-34011-hadoop-hdfs-3_4_1-tests_jar-_-any-18191241740691095835/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T10:23:42,799 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:34011} 2024-11-16T10:23:42,799 INFO [Time-limited test {}] server.Server(415): Started @3461ms 2024-11-16T10:23:43,510 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T10:23:43,518 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T10:23:43,519 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T10:23:43,519 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T10:23:43,520 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-16T10:23:43,520 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/hadoop.log.dir/,AVAILABLE} 2024-11-16T10:23:43,521 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T10:23:43,645 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/java.io.tmpdir/jetty-localhost-34283-hadoop-hdfs-3_4_1-tests_jar-_-any-10474331581521288812/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T10:23:43,646 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:34283} 2024-11-16T10:23:43,646 INFO [Time-limited test {}] server.Server(415): Started @4308ms 2024-11-16T10:23:43,701 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-16T10:23:44,936 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/cluster_7ec35abe-b1d1-d9da-1787-50b9d114d89d/data/data2/current/BP-1320990832-172.17.0.2-1731752621574/current, will proceed with Du for space computation calculation, 2024-11-16T10:23:44,936 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/cluster_7ec35abe-b1d1-d9da-1787-50b9d114d89d/data/data1/current/BP-1320990832-172.17.0.2-1731752621574/current, will proceed with Du for space computation calculation, 2024-11-16T10:23:44,972 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-16T10:23:45,031 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb3e5e18ba6254cf8 with lease ID 0xbd23a35daf64d10: Processing first storage report for DS-129f4add-b7b1-47f0-bebd-7b8145511656 from datanode DatanodeRegistration(127.0.0.1:41801, datanodeUuid=db4eb425-94e6-4ce5-acf1-af055905740e, infoPort=40561, infoSecurePort=0, ipcPort=43719, storageInfo=lv=-57;cid=testClusterID;nsid=93254438;c=1731752621574) 2024-11-16T10:23:45,032 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb3e5e18ba6254cf8 with lease ID 0xbd23a35daf64d10: from storage DS-129f4add-b7b1-47f0-bebd-7b8145511656 node DatanodeRegistration(127.0.0.1:41801, datanodeUuid=db4eb425-94e6-4ce5-acf1-af055905740e, infoPort=40561, infoSecurePort=0, ipcPort=43719, storageInfo=lv=-57;cid=testClusterID;nsid=93254438;c=1731752621574), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-16T10:23:45,032 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb3e5e18ba6254cf8 with lease ID 0xbd23a35daf64d10: Processing first storage report for DS-fec5cec5-7a99-42a0-b717-3997922da924 from datanode DatanodeRegistration(127.0.0.1:41801, datanodeUuid=db4eb425-94e6-4ce5-acf1-af055905740e, infoPort=40561, infoSecurePort=0, ipcPort=43719, storageInfo=lv=-57;cid=testClusterID;nsid=93254438;c=1731752621574) 2024-11-16T10:23:45,033 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb3e5e18ba6254cf8 with lease ID 0xbd23a35daf64d10: from storage DS-fec5cec5-7a99-42a0-b717-3997922da924 node DatanodeRegistration(127.0.0.1:41801, datanodeUuid=db4eb425-94e6-4ce5-acf1-af055905740e, infoPort=40561, infoSecurePort=0, ipcPort=43719, storageInfo=lv=-57;cid=testClusterID;nsid=93254438;c=1731752621574), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T10:23:45,098 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11 2024-11-16T10:23:45,165 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/cluster_7ec35abe-b1d1-d9da-1787-50b9d114d89d/zookeeper_0, clientPort=49279, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/cluster_7ec35abe-b1d1-d9da-1787-50b9d114d89d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/cluster_7ec35abe-b1d1-d9da-1787-50b9d114d89d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-16T10:23:45,176 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49279 2024-11-16T10:23:45,185 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T10:23:45,188 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T10:23:45,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741825_1001 (size=7) 2024-11-16T10:23:45,838 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7 with version=8 2024-11-16T10:23:45,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/hbase-staging 2024-11-16T10:23:45,953 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-16T10:23:46,221 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b16ff38458c1:0 server-side Connection retries=45 2024-11-16T10:23:46,232 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T10:23:46,233 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T10:23:46,237 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T10:23:46,238 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T10:23:46,238 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T10:23:46,432 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T10:23:46,499 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-16T10:23:46,508 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-16T10:23:46,511 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T10:23:46,535 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 33208 (auto-detected) 2024-11-16T10:23:46,536 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-16T10:23:46,557 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38809 2024-11-16T10:23:46,580 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38809 connecting to ZooKeeper ensemble=127.0.0.1:49279 2024-11-16T10:23:46,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:388090x0, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T10:23:46,831 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38809-0x101432bffa20000 connected 2024-11-16T10:23:47,389 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T10:23:47,396 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T10:23:47,404 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T10:23:47,409 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7, hbase.cluster.distributed=false 2024-11-16T10:23:47,431 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T10:23:47,435 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38809 2024-11-16T10:23:47,436 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38809 2024-11-16T10:23:47,436 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38809 2024-11-16T10:23:47,437 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38809 2024-11-16T10:23:47,437 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38809 2024-11-16T10:23:47,536 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b16ff38458c1:0 server-side Connection retries=45 2024-11-16T10:23:47,538 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T10:23:47,538 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T10:23:47,538 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T10:23:47,538 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T10:23:47,539 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T10:23:47,541 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T10:23:47,544 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T10:23:47,544 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35873 2024-11-16T10:23:47,546 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35873 connecting to ZooKeeper ensemble=127.0.0.1:49279 2024-11-16T10:23:47,547 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T10:23:47,550 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T10:23:47,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:358730x0, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T10:23:47,625 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35873-0x101432bffa20001 connected 2024-11-16T10:23:47,625 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35873-0x101432bffa20001, quorum=127.0.0.1:49279, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T10:23:47,630 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T10:23:47,638 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T10:23:47,641 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35873-0x101432bffa20001, quorum=127.0.0.1:49279, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T10:23:47,646 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35873-0x101432bffa20001, quorum=127.0.0.1:49279, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T10:23:47,646 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35873 2024-11-16T10:23:47,647 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35873 2024-11-16T10:23:47,647 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35873 2024-11-16T10:23:47,648 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35873 2024-11-16T10:23:47,648 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35873 2024-11-16T10:23:47,666 DEBUG [M:0;b16ff38458c1:38809 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b16ff38458c1:38809 2024-11-16T10:23:47,667 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b16ff38458c1,38809,1731752626047 2024-11-16T10:23:47,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T10:23:47,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35873-0x101432bffa20001, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T10:23:47,796 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b16ff38458c1,38809,1731752626047 2024-11-16T10:23:47,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:23:47,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35873-0x101432bffa20001, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T10:23:47,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35873-0x101432bffa20001, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:23:47,986 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T10:23:47,987 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b16ff38458c1,38809,1731752626047 from backup master directory 2024-11-16T10:23:48,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b16ff38458c1,38809,1731752626047 2024-11-16T10:23:48,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T10:23:48,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35873-0x101432bffa20001, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T10:23:48,161 WARN [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T10:23:48,161 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b16ff38458c1,38809,1731752626047 2024-11-16T10:23:48,164 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-16T10:23:48,165 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-16T10:23:48,227 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/hbase.id] with ID: 44c99309-bfed-483a-a470-cc4f96bfb762 2024-11-16T10:23:48,227 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/.tmp/hbase.id 2024-11-16T10:23:48,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741826_1002 (size=42) 2024-11-16T10:23:48,248 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/.tmp/hbase.id]:[hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/hbase.id] 2024-11-16T10:23:48,320 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T10:23:48,327 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T10:23:48,345 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-16T10:23:48,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:23:48,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35873-0x101432bffa20001, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:23:48,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741827_1003 (size=196) 2024-11-16T10:23:48,516 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T10:23:48,518 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T10:23:48,531 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:48,535 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T10:23:48,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741828_1004 (size=1189) 2024-11-16T10:23:48,986 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/MasterData/data/master/store 2024-11-16T10:23:49,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741829_1005 (size=34) 2024-11-16T10:23:49,416 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-16T10:23:49,421 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:23:49,423 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T10:23:49,423 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T10:23:49,423 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T10:23:49,425 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T10:23:49,426 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T10:23:49,426 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T10:23:49,428 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731752629422Disabling compacts and flushes for region at 1731752629422Disabling writes for close at 1731752629426 (+4 ms)Writing region close event to WAL at 1731752629426Closed at 1731752629426 2024-11-16T10:23:49,431 WARN [master/b16ff38458c1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/MasterData/data/master/store/.initializing 2024-11-16T10:23:49,431 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/MasterData/WALs/b16ff38458c1,38809,1731752626047 2024-11-16T10:23:49,441 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T10:23:49,461 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b16ff38458c1%2C38809%2C1731752626047, suffix=, logDir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/MasterData/WALs/b16ff38458c1,38809,1731752626047, archiveDir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/MasterData/oldWALs, maxLogs=10 2024-11-16T10:23:49,493 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/MasterData/WALs/b16ff38458c1,38809,1731752626047/b16ff38458c1%2C38809%2C1731752626047.1731752629467, exclude list is [], retry=0 2024-11-16T10:23:49,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41801,DS-129f4add-b7b1-47f0-bebd-7b8145511656,DISK] 2024-11-16T10:23:49,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-16T10:23:49,569 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/MasterData/WALs/b16ff38458c1,38809,1731752626047/b16ff38458c1%2C38809%2C1731752626047.1731752629467 2024-11-16T10:23:49,571 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40561:40561)] 2024-11-16T10:23:49,571 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T10:23:49,572 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:23:49,578 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T10:23:49,579 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T10:23:49,633 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T10:23:49,665 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T10:23:49,670 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:23:49,673 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T10:23:49,674 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T10:23:49,678 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T10:23:49,679 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:23:49,680 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:23:49,681 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T10:23:49,684 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T10:23:49,684 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:23:49,685 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:23:49,686 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T10:23:49,689 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T10:23:49,689 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:23:49,690 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:23:49,691 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T10:23:49,695 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T10:23:49,696 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T10:23:49,703 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T10:23:49,704 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T10:23:49,708 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T10:23:49,713 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T10:23:49,718 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T10:23:49,720 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62693729, jitterRate=-0.06579063832759857}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T10:23:49,728 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731752629595Initializing all the Stores at 1731752629598 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731752629599 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752629600 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752629601 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752629601Cleaning up temporary data from old regions at 1731752629704 (+103 ms)Region opened successfully at 1731752629728 (+24 ms) 2024-11-16T10:23:49,729 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T10:23:49,765 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29efa2cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b16ff38458c1/172.17.0.2:0 2024-11-16T10:23:49,802 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-16T10:23:49,814 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-16T10:23:49,815 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-16T10:23:49,817 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T10:23:49,819 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-16T10:23:49,824 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-16T10:23:49,824 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T10:23:49,854 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-16T10:23:49,866 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T10:23:49,928 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T10:23:49,931 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T10:23:49,933 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T10:23:49,939 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T10:23:49,942 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T10:23:49,946 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T10:23:49,959 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T10:23:49,960 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T10:23:49,970 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T10:23:49,989 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T10:23:50,002 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T10:23:50,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35873-0x101432bffa20001, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T10:23:50,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T10:23:50,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35873-0x101432bffa20001, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:23:50,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:23:50,017 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=b16ff38458c1,38809,1731752626047, sessionid=0x101432bffa20000, setting cluster-up flag (Was=false) 2024-11-16T10:23:50,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:23:50,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35873-0x101432bffa20001, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:23:50,076 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T10:23:50,078 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b16ff38458c1,38809,1731752626047 2024-11-16T10:23:50,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35873-0x101432bffa20001, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:23:50,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:23:50,128 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T10:23:50,130 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b16ff38458c1,38809,1731752626047 2024-11-16T10:23:50,136 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-16T10:23:50,154 INFO [RS:0;b16ff38458c1:35873 {}] regionserver.HRegionServer(746): ClusterId : 44c99309-bfed-483a-a470-cc4f96bfb762 2024-11-16T10:23:50,157 DEBUG [RS:0;b16ff38458c1:35873 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T10:23:50,173 DEBUG [RS:0;b16ff38458c1:35873 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T10:23:50,173 DEBUG [RS:0;b16ff38458c1:35873 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T10:23:50,182 DEBUG [RS:0;b16ff38458c1:35873 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T10:23:50,183 DEBUG [RS:0;b16ff38458c1:35873 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14313d5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b16ff38458c1/172.17.0.2:0 2024-11-16T10:23:50,199 DEBUG [RS:0;b16ff38458c1:35873 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b16ff38458c1:35873 2024-11-16T10:23:50,203 INFO [RS:0;b16ff38458c1:35873 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T10:23:50,203 INFO [RS:0;b16ff38458c1:35873 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T10:23:50,204 DEBUG [RS:0;b16ff38458c1:35873 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T10:23:50,209 INFO [RS:0;b16ff38458c1:35873 {}] regionserver.HRegionServer(2659): reportForDuty to master=b16ff38458c1,38809,1731752626047 with port=35873, startcode=1731752627506 2024-11-16T10:23:50,222 DEBUG [RS:0;b16ff38458c1:35873 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T10:23:50,231 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T10:23:50,242 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T10:23:50,250 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-16T10:23:50,256 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b16ff38458c1,38809,1731752626047 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T10:23:50,264 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59721, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T10:23:50,265 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b16ff38458c1:0, corePoolSize=5, maxPoolSize=5 2024-11-16T10:23:50,265 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b16ff38458c1:0, corePoolSize=5, maxPoolSize=5 2024-11-16T10:23:50,265 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b16ff38458c1:0, corePoolSize=5, maxPoolSize=5 2024-11-16T10:23:50,265 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b16ff38458c1:0, corePoolSize=5, maxPoolSize=5 2024-11-16T10:23:50,266 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b16ff38458c1:0, corePoolSize=10, maxPoolSize=10 2024-11-16T10:23:50,266 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b16ff38458c1:0, corePoolSize=1, maxPoolSize=1 2024-11-16T10:23:50,266 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b16ff38458c1:0, corePoolSize=2, maxPoolSize=2 2024-11-16T10:23:50,266 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b16ff38458c1:0, corePoolSize=1, maxPoolSize=1 2024-11-16T10:23:50,271 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38809 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:50,285 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T10:23:50,286 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-16T10:23:50,288 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731752660288 2024-11-16T10:23:50,290 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T10:23:50,291 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T10:23:50,294 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:23:50,295 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T10:23:50,295 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-16T10:23:50,295 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T10:23:50,296 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T10:23:50,296 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T10:23:50,297 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T10:23:50,303 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T10:23:50,304 DEBUG [RS:0;b16ff38458c1:35873 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-16T10:23:50,304 WARN [RS:0;b16ff38458c1:35873 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-16T10:23:50,305 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T10:23:50,306 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T10:23:50,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741831_1007 (size=1321) 2024-11-16T10:23:50,312 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T10:23:50,312 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-16T10:23:50,312 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T10:23:50,313 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7 2024-11-16T10:23:50,315 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b16ff38458c1:0:becomeActiveMaster-HFileCleaner.large.0-1731752630314,5,FailOnTimeoutGroup] 2024-11-16T10:23:50,315 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b16ff38458c1:0:becomeActiveMaster-HFileCleaner.small.0-1731752630315,5,FailOnTimeoutGroup] 2024-11-16T10:23:50,316 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T10:23:50,316 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T10:23:50,317 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T10:23:50,318 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T10:23:50,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741832_1008 (size=32) 2024-11-16T10:23:50,406 INFO [RS:0;b16ff38458c1:35873 {}] regionserver.HRegionServer(2659): reportForDuty to master=b16ff38458c1,38809,1731752626047 with port=35873, startcode=1731752627506 2024-11-16T10:23:50,408 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38809 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b16ff38458c1,35873,1731752627506 2024-11-16T10:23:50,412 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38809 {}] master.ServerManager(517): Registering regionserver=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:50,422 DEBUG [RS:0;b16ff38458c1:35873 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7 2024-11-16T10:23:50,422 DEBUG [RS:0;b16ff38458c1:35873 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41487 2024-11-16T10:23:50,423 DEBUG [RS:0;b16ff38458c1:35873 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T10:23:50,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T10:23:50,435 DEBUG [RS:0;b16ff38458c1:35873 {}] zookeeper.ZKUtil(111): regionserver:35873-0x101432bffa20001, quorum=127.0.0.1:49279, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b16ff38458c1,35873,1731752627506 2024-11-16T10:23:50,435 WARN [RS:0;b16ff38458c1:35873 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T10:23:50,435 INFO [RS:0;b16ff38458c1:35873 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T10:23:50,436 DEBUG [RS:0;b16ff38458c1:35873 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/WALs/b16ff38458c1,35873,1731752627506 2024-11-16T10:23:50,437 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b16ff38458c1,35873,1731752627506] 2024-11-16T10:23:50,462 INFO [RS:0;b16ff38458c1:35873 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T10:23:50,481 INFO [RS:0;b16ff38458c1:35873 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T10:23:50,488 INFO [RS:0;b16ff38458c1:35873 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T10:23:50,488 INFO [RS:0;b16ff38458c1:35873 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T10:23:50,489 INFO [RS:0;b16ff38458c1:35873 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T10:23:50,496 INFO [RS:0;b16ff38458c1:35873 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T10:23:50,498 INFO [RS:0;b16ff38458c1:35873 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T10:23:50,499 DEBUG [RS:0;b16ff38458c1:35873 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b16ff38458c1:0, corePoolSize=1, maxPoolSize=1 2024-11-16T10:23:50,499 DEBUG [RS:0;b16ff38458c1:35873 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b16ff38458c1:0, corePoolSize=1, maxPoolSize=1 2024-11-16T10:23:50,499 DEBUG [RS:0;b16ff38458c1:35873 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b16ff38458c1:0, corePoolSize=1, maxPoolSize=1 2024-11-16T10:23:50,499 DEBUG [RS:0;b16ff38458c1:35873 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b16ff38458c1:0, corePoolSize=1, maxPoolSize=1 2024-11-16T10:23:50,499 DEBUG [RS:0;b16ff38458c1:35873 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b16ff38458c1:0, corePoolSize=1, maxPoolSize=1 2024-11-16T10:23:50,500 DEBUG [RS:0;b16ff38458c1:35873 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b16ff38458c1:0, corePoolSize=2, maxPoolSize=2 2024-11-16T10:23:50,500 DEBUG [RS:0;b16ff38458c1:35873 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0, corePoolSize=1, maxPoolSize=1 2024-11-16T10:23:50,500 DEBUG [RS:0;b16ff38458c1:35873 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b16ff38458c1:0, corePoolSize=1, maxPoolSize=1 2024-11-16T10:23:50,500 DEBUG [RS:0;b16ff38458c1:35873 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b16ff38458c1:0, corePoolSize=1, maxPoolSize=1 2024-11-16T10:23:50,500 DEBUG [RS:0;b16ff38458c1:35873 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b16ff38458c1:0, corePoolSize=1, maxPoolSize=1 2024-11-16T10:23:50,500 DEBUG [RS:0;b16ff38458c1:35873 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b16ff38458c1:0, corePoolSize=1, maxPoolSize=1 2024-11-16T10:23:50,501 DEBUG [RS:0;b16ff38458c1:35873 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b16ff38458c1:0, corePoolSize=1, maxPoolSize=1 2024-11-16T10:23:50,501 DEBUG [RS:0;b16ff38458c1:35873 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b16ff38458c1:0, corePoolSize=3, maxPoolSize=3 2024-11-16T10:23:50,501 DEBUG [RS:0;b16ff38458c1:35873 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0, corePoolSize=3, maxPoolSize=3 2024-11-16T10:23:50,503 INFO [RS:0;b16ff38458c1:35873 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T10:23:50,504 INFO [RS:0;b16ff38458c1:35873 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T10:23:50,504 INFO [RS:0;b16ff38458c1:35873 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T10:23:50,504 INFO [RS:0;b16ff38458c1:35873 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T10:23:50,504 INFO [RS:0;b16ff38458c1:35873 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T10:23:50,504 INFO [RS:0;b16ff38458c1:35873 {}] hbase.ChoreService(168): Chore ScheduledChore name=b16ff38458c1,35873,1731752627506-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T10:23:50,528 INFO [RS:0;b16ff38458c1:35873 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T10:23:50,530 INFO [RS:0;b16ff38458c1:35873 {}] hbase.ChoreService(168): Chore ScheduledChore name=b16ff38458c1,35873,1731752627506-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T10:23:50,531 INFO [RS:0;b16ff38458c1:35873 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T10:23:50,531 INFO [RS:0;b16ff38458c1:35873 {}] regionserver.Replication(171): b16ff38458c1,35873,1731752627506 started 2024-11-16T10:23:50,557 INFO [RS:0;b16ff38458c1:35873 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T10:23:50,557 INFO [RS:0;b16ff38458c1:35873 {}] regionserver.HRegionServer(1482): Serving as b16ff38458c1,35873,1731752627506, RpcServer on b16ff38458c1/172.17.0.2:35873, sessionid=0x101432bffa20001 2024-11-16T10:23:50,558 DEBUG [RS:0;b16ff38458c1:35873 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T10:23:50,558 DEBUG [RS:0;b16ff38458c1:35873 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b16ff38458c1,35873,1731752627506 2024-11-16T10:23:50,559 DEBUG [RS:0;b16ff38458c1:35873 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b16ff38458c1,35873,1731752627506' 2024-11-16T10:23:50,559 DEBUG [RS:0;b16ff38458c1:35873 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T10:23:50,560 DEBUG [RS:0;b16ff38458c1:35873 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T10:23:50,561 DEBUG [RS:0;b16ff38458c1:35873 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T10:23:50,561 DEBUG [RS:0;b16ff38458c1:35873 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T10:23:50,561 DEBUG [RS:0;b16ff38458c1:35873 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b16ff38458c1,35873,1731752627506 2024-11-16T10:23:50,561 DEBUG [RS:0;b16ff38458c1:35873 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b16ff38458c1,35873,1731752627506' 2024-11-16T10:23:50,561 DEBUG [RS:0;b16ff38458c1:35873 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T10:23:50,562 DEBUG [RS:0;b16ff38458c1:35873 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T10:23:50,562 DEBUG [RS:0;b16ff38458c1:35873 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T10:23:50,563 INFO [RS:0;b16ff38458c1:35873 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T10:23:50,563 INFO [RS:0;b16ff38458c1:35873 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T10:23:50,667 INFO [RS:0;b16ff38458c1:35873 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T10:23:50,670 INFO [RS:0;b16ff38458c1:35873 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b16ff38458c1%2C35873%2C1731752627506, suffix=, logDir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/WALs/b16ff38458c1,35873,1731752627506, archiveDir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/oldWALs, maxLogs=32 2024-11-16T10:23:50,685 DEBUG [RS:0;b16ff38458c1:35873 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/WALs/b16ff38458c1,35873,1731752627506/b16ff38458c1%2C35873%2C1731752627506.1731752630673, exclude list is [], retry=0 2024-11-16T10:23:50,689 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41801,DS-129f4add-b7b1-47f0-bebd-7b8145511656,DISK] 2024-11-16T10:23:50,692 INFO [RS:0;b16ff38458c1:35873 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/WALs/b16ff38458c1,35873,1731752627506/b16ff38458c1%2C35873%2C1731752627506.1731752630673 2024-11-16T10:23:50,693 DEBUG [RS:0;b16ff38458c1:35873 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40561:40561)] 2024-11-16T10:23:50,729 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:23:50,733 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T10:23:50,738 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T10:23:50,738 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:23:50,740 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T10:23:50,740 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T10:23:50,744 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T10:23:50,744 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:23:50,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T10:23:50,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T10:23:50,748 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T10:23:50,748 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:23:50,749 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T10:23:50,749 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T10:23:50,752 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T10:23:50,752 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:23:50,752 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T10:23:50,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T10:23:50,754 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/hbase/meta/1588230740 2024-11-16T10:23:50,755 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/hbase/meta/1588230740 2024-11-16T10:23:50,758 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T10:23:50,758 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T10:23:50,758 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T10:23:50,761 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T10:23:50,765 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T10:23:50,767 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62166630, jitterRate=-0.07364502549171448}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T10:23:50,769 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731752630730Initializing all the Stores at 1731752630732 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731752630733 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731752630733Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752630733Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731752630733Cleaning up temporary data from old regions at 1731752630758 (+25 ms)Region opened successfully at 1731752630769 (+11 ms) 2024-11-16T10:23:50,769 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T10:23:50,769 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T10:23:50,769 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T10:23:50,769 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T10:23:50,770 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T10:23:50,771 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T10:23:50,771 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731752630769Disabling compacts and flushes for region at 1731752630769Disabling writes for close at 1731752630770 (+1 ms)Writing region close event to WAL at 1731752630770Closed at 1731752630771 (+1 ms) 2024-11-16T10:23:50,774 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T10:23:50,774 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-16T10:23:50,780 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T10:23:50,786 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T10:23:50,789 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-16T10:23:50,945 DEBUG [b16ff38458c1:38809 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-16T10:23:50,959 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:50,965 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b16ff38458c1,35873,1731752627506, state=OPENING 2024-11-16T10:23:51,012 DEBUG [PEWorker-4 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-16T10:23:51,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35873-0x101432bffa20001, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:23:51,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:23:51,024 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T10:23:51,024 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T10:23:51,026 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T10:23:51,028 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b16ff38458c1,35873,1731752627506}] 2024-11-16T10:23:51,203 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T10:23:51,206 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38611, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T10:23:51,216 INFO [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T10:23:51,217 INFO [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T10:23:51,217 INFO [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T10:23:51,220 INFO [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b16ff38458c1%2C35873%2C1731752627506.meta, suffix=.meta, logDir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/WALs/b16ff38458c1,35873,1731752627506, archiveDir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/oldWALs, maxLogs=32 2024-11-16T10:23:51,235 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/WALs/b16ff38458c1,35873,1731752627506/b16ff38458c1%2C35873%2C1731752627506.meta.1731752631222.meta, exclude list is [], retry=0 2024-11-16T10:23:51,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41801,DS-129f4add-b7b1-47f0-bebd-7b8145511656,DISK] 2024-11-16T10:23:51,241 INFO [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/WALs/b16ff38458c1,35873,1731752627506/b16ff38458c1%2C35873%2C1731752627506.meta.1731752631222.meta 2024-11-16T10:23:51,242 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40561:40561)] 2024-11-16T10:23:51,242 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T10:23:51,243 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T10:23:51,246 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T10:23:51,249 INFO [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T10:23:51,253 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T10:23:51,254 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:23:51,254 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T10:23:51,254 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T10:23:51,257 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T10:23:51,258 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T10:23:51,258 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:23:51,259 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T10:23:51,259 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T10:23:51,260 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T10:23:51,260 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:23:51,261 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T10:23:51,261 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T10:23:51,263 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T10:23:51,263 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:23:51,263 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T10:23:51,264 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T10:23:51,265 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T10:23:51,265 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:23:51,266 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T10:23:51,266 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T10:23:51,267 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/hbase/meta/1588230740 2024-11-16T10:23:51,270 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/hbase/meta/1588230740 2024-11-16T10:23:51,272 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T10:23:51,272 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T10:23:51,273 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T10:23:51,276 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T10:23:51,278 INFO [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75107572, jitterRate=0.1191900372505188}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T10:23:51,278 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T10:23:51,280 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731752631254Writing region info on filesystem at 1731752631254Initializing all the Stores at 1731752631256 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731752631256Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731752631257 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752631257Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731752631257Cleaning up temporary data from old regions at 1731752631272 (+15 ms)Running coprocessor post-open hooks at 1731752631278 (+6 ms)Region opened successfully at 1731752631280 (+2 ms) 2024-11-16T10:23:51,287 INFO [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731752631196 2024-11-16T10:23:51,296 DEBUG [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T10:23:51,297 INFO [RS_OPEN_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T10:23:51,298 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:51,300 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b16ff38458c1,35873,1731752627506, state=OPEN 2024-11-16T10:23:51,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35873-0x101432bffa20001, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T10:23:51,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T10:23:51,358 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T10:23:51,358 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T10:23:51,358 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:51,366 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-16T10:23:51,366 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b16ff38458c1,35873,1731752627506 in 331 msec 2024-11-16T10:23:51,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-16T10:23:51,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 590 msec 2024-11-16T10:23:51,378 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T10:23:51,378 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-16T10:23:51,394 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:23:51,396 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:23:51,413 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:23:51,415 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54581, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:23:51,435 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2550 sec 2024-11-16T10:23:51,436 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731752631436, completionTime=-1 2024-11-16T10:23:51,438 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-16T10:23:51,439 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-16T10:23:51,485 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-16T10:23:51,485 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731752691485 2024-11-16T10:23:51,485 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731752751485 2024-11-16T10:23:51,485 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 46 msec 2024-11-16T10:23:51,488 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b16ff38458c1,38809,1731752626047-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T10:23:51,488 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b16ff38458c1,38809,1731752626047-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T10:23:51,488 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b16ff38458c1,38809,1731752626047-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T10:23:51,490 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b16ff38458c1:38809, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T10:23:51,491 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T10:23:51,491 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T10:23:51,499 DEBUG [master/b16ff38458c1:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-16T10:23:51,527 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.365sec 2024-11-16T10:23:51,528 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T10:23:51,529 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T10:23:51,530 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T10:23:51,531 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T10:23:51,531 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T10:23:51,532 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b16ff38458c1,38809,1731752626047-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T10:23:51,532 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b16ff38458c1,38809,1731752626047-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T10:23:51,540 DEBUG [master/b16ff38458c1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T10:23:51,541 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T10:23:51,542 INFO [master/b16ff38458c1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b16ff38458c1,38809,1731752626047-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T10:23:51,565 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:51,567 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-16T10:23:51,567 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-16T10:23:51,570 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:23:51,572 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:23:51,579 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:23:51,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:23:51,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:23:51,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5db9d8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:51,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:23:51,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:23:51,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:23:51,589 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:23:51,592 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:51,592 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:23:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:23:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:23:51,602 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:23:51,620 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=b16ff38458c1,38809,1731752626047 2024-11-16T10:23:51,630 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=396, ProcessCount=11, AvailableMemoryMB=5039 2024-11-16T10:23:51,638 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T10:23:51,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is b16ff38458c1,38809,1731752626047 2024-11-16T10:23:51,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5b65baa9 2024-11-16T10:23:51,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T10:23:51,649 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T10:23:51,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T10:23:51,655 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T10:23:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T10:23:51,664 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T10:23:51,666 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-16T10:23:51,666 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:23:51,669 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T10:23:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T10:23:51,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741835_1011 (size=959) 2024-11-16T10:23:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T10:23:51,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T10:23:52,085 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7 2024-11-16T10:23:52,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741836_1012 (size=53) 2024-11-16T10:23:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T10:23:52,498 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:23:52,499 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e63c645d595c80b0e763eedbebd195b2, disabling compactions & flushes 2024-11-16T10:23:52,499 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:52,499 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:52,499 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. after waiting 0 ms 2024-11-16T10:23:52,499 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:52,499 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:52,499 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e63c645d595c80b0e763eedbebd195b2: Waiting for close lock at 1731752632499Disabling compacts and flushes for region at 1731752632499Disabling writes for close at 1731752632499Writing region close event to WAL at 1731752632499Closed at 1731752632499 2024-11-16T10:23:52,502 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T10:23:52,508 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731752632502"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731752632502"}]},"ts":"1731752632502"} 2024-11-16T10:23:52,513 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T10:23:52,515 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T10:23:52,518 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752632515"}]},"ts":"1731752632515"} 2024-11-16T10:23:52,522 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T10:23:52,524 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e63c645d595c80b0e763eedbebd195b2, ASSIGN}] 2024-11-16T10:23:52,526 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e63c645d595c80b0e763eedbebd195b2, ASSIGN 2024-11-16T10:23:52,528 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e63c645d595c80b0e763eedbebd195b2, ASSIGN; state=OFFLINE, location=b16ff38458c1,35873,1731752627506; forceNewPlan=false, retain=false 2024-11-16T10:23:52,681 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e63c645d595c80b0e763eedbebd195b2, regionState=OPENING, regionLocation=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:52,692 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e63c645d595c80b0e763eedbebd195b2, ASSIGN because future has completed 2024-11-16T10:23:52,693 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506}] 2024-11-16T10:23:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T10:23:52,860 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:52,860 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} 2024-11-16T10:23:52,860 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:23:52,860 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:23:52,861 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:23:52,861 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:23:52,863 INFO [StoreOpener-e63c645d595c80b0e763eedbebd195b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:23:52,866 INFO [StoreOpener-e63c645d595c80b0e763eedbebd195b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e63c645d595c80b0e763eedbebd195b2 columnFamilyName A 2024-11-16T10:23:52,866 DEBUG [StoreOpener-e63c645d595c80b0e763eedbebd195b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:23:52,867 INFO [StoreOpener-e63c645d595c80b0e763eedbebd195b2-1 {}] regionserver.HStore(327): Store=e63c645d595c80b0e763eedbebd195b2/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:23:52,868 INFO [StoreOpener-e63c645d595c80b0e763eedbebd195b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:23:52,870 INFO [StoreOpener-e63c645d595c80b0e763eedbebd195b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e63c645d595c80b0e763eedbebd195b2 columnFamilyName B 2024-11-16T10:23:52,870 DEBUG [StoreOpener-e63c645d595c80b0e763eedbebd195b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:23:52,871 INFO [StoreOpener-e63c645d595c80b0e763eedbebd195b2-1 {}] regionserver.HStore(327): Store=e63c645d595c80b0e763eedbebd195b2/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:23:52,871 INFO [StoreOpener-e63c645d595c80b0e763eedbebd195b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:23:52,874 INFO [StoreOpener-e63c645d595c80b0e763eedbebd195b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e63c645d595c80b0e763eedbebd195b2 columnFamilyName C 2024-11-16T10:23:52,874 DEBUG [StoreOpener-e63c645d595c80b0e763eedbebd195b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:23:52,874 INFO [StoreOpener-e63c645d595c80b0e763eedbebd195b2-1 {}] regionserver.HStore(327): Store=e63c645d595c80b0e763eedbebd195b2/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:23:52,875 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:23:52,876 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:23:52,877 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:23:52,879 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:23:52,879 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:23:52,880 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T10:23:52,882 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:23:52,886 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T10:23:52,886 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e63c645d595c80b0e763eedbebd195b2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60634950, jitterRate=-0.09646883606910706}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T10:23:52,887 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:23:52,887 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e63c645d595c80b0e763eedbebd195b2: Running coprocessor pre-open hook at 1731752632861Writing region info on filesystem at 1731752632861Initializing all the Stores at 1731752632862 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752632863 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752632863Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752632863Cleaning up temporary data from old regions at 1731752632879 (+16 ms)Running coprocessor post-open hooks at 1731752632887 (+8 ms)Region opened successfully at 1731752632887 2024-11-16T10:23:52,889 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., pid=6, masterSystemTime=1731752632847 2024-11-16T10:23:52,893 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:52,893 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:52,894 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e63c645d595c80b0e763eedbebd195b2, regionState=OPEN, openSeqNum=2, regionLocation=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:52,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 because future has completed 2024-11-16T10:23:52,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-16T10:23:52,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 in 207 msec 2024-11-16T10:23:52,910 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-16T10:23:52,910 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e63c645d595c80b0e763eedbebd195b2, ASSIGN in 382 msec 2024-11-16T10:23:52,912 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T10:23:52,913 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752632912"}]},"ts":"1731752632912"} 2024-11-16T10:23:52,916 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T10:23:52,918 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T10:23:52,922 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2610 sec 2024-11-16T10:23:53,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T10:23:53,829 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:23:53,838 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,838 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:23:53,839 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:23:53,841 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:23:53,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:23:53,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:23:53,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e00daac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:23:53,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:23:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:23:53,845 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:23:53,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f43f46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,848 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:23:53,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:23:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:23:53,852 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:23:53,854 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T10:23:53,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is b16ff38458c1,38809,1731752626047 2024-11-16T10:23:53,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1907f2c6 2024-11-16T10:23:53,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T10:23:53,860 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T10:23:53,865 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,865 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:23:53,865 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:23:53,867 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:23:53,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:23:53,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:23:53,868 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@387dfdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,868 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:23:53,868 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:23:53,869 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:23:53,870 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:23:53,871 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25175df8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,872 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,872 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:23:53,873 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:23:53,874 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:23:53,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:23:53,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:23:53,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e1a1116, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:23:53,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:23:53,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:23:53,877 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:23:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e59e9dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,880 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:23:53,881 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:23:53,882 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:23:53,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:23:53,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:23:53,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b17c100, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:23:53,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:23:53,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:23:53,886 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:23:53,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4184659e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,888 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:23:53,888 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:23:53,890 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:23:53,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:23:53,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:23:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a183203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:23:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:23:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:23:53,893 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:23:53,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fd2fdd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,895 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:23:53,895 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:23:53,896 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:23:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:23:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:23:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9a6bf59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:23:53,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:23:53,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:23:53,899 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:23:53,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54c50a81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,902 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:23:53,902 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:23:53,904 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:23:53,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:23:53,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:23:53,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@700b90fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:23:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:23:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:23:53,907 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:23:53,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cb8306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,909 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,909 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:23:53,909 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:23:53,910 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:23:53,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:23:53,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:23:53,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@94111c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:23:53,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:23:53,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:23:53,913 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:23:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13185de7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,915 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:23:53,915 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:23:53,917 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:23:53,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:23:53,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:23:53,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b456016, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:23:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:23:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:23:53,919 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:23:53,921 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cc2de1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,922 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,922 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:23:53,922 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:23:53,923 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:23:53,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:23:53,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:23:53,924 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7994a976, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,924 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:23:53,924 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:23:53,924 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:23:53,925 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:23:53,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7028084d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:23:53,932 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:23:53,933 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:23:53,933 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:23:53,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:23:53,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:23:53,934 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:23:53,935 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:23:53,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:23:53,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:23:53,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:23:53,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:23:53,936 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:23:53,936 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:23:53,937 DEBUG [Thread-147 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:23:53,937 DEBUG [Thread-147 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:23:53,938 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:23:53,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:23:53,938 DEBUG [Thread-154 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:23:53,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:23:53,939 DEBUG [Thread-154 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:23:53,939 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:23:53,939 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:23:53,941 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:23:53,941 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:23:53,942 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:23:53,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:23:53,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:23:53,943 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:23:53,943 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:23:53,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:23:53,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:23:53,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:23:53,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:23:53,946 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:23:53,946 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:23:53,946 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:23:53,947 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:23:53,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2] 2024-11-16T10:23:53,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2] 2024-11-16T10:23:53,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2] 2024-11-16T10:23:53,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2] 2024-11-16T10:23:53,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2] 2024-11-16T10:23:53,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2] 2024-11-16T10:23:53,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2] 2024-11-16T10:23:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-16T10:23:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2] 2024-11-16T10:23:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T10:23:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2] 2024-11-16T10:23:53,964 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:23:53,968 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:23:53,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:23:54,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:23:54,030 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:23:54,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e63c645d595c80b0e763eedbebd195b2 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T10:23:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T10:23:54,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T10:23:54,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:54,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/0b5b99bc0f1846289d0c50f9fda71e76 is 50, key is test_row_0/A:col10/1731752633958/Put/seqid=0 2024-11-16T10:23:54,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:23:54,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:54,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:54,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:54,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741837_1013 (size=16681) 2024-11-16T10:23:54,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/0b5b99bc0f1846289d0c50f9fda71e76 2024-11-16T10:23:54,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731752694183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:54,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752694189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:54,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:54,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752694194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:54,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:54,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37652 deadline: 1731752694194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:54,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:54,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752694201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:54,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T10:23:54,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/1fbae99c4d214a67be37be4105945ca5 is 50, key is test_row_0/B:col10/1731752633958/Put/seqid=0 2024-11-16T10:23:54,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752694327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:54,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T10:23:54,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:54,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752694328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:54,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:54,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731752694328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:54,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:54,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37652 deadline: 1731752694332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:54,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741838_1014 (size=12001) 2024-11-16T10:23:54,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:54,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/1fbae99c4d214a67be37be4105945ca5 2024-11-16T10:23:54,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:54,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:23:54,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:54,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:54,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:54,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752694338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:54,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:54,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/61216607784a47a5ace23c975309c00b is 50, key is test_row_0/C:col10/1731752633958/Put/seqid=0 2024-11-16T10:23:54,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741839_1015 (size=12001) 2024-11-16T10:23:54,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/61216607784a47a5ace23c975309c00b 2024-11-16T10:23:54,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/0b5b99bc0f1846289d0c50f9fda71e76 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/0b5b99bc0f1846289d0c50f9fda71e76 2024-11-16T10:23:54,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/0b5b99bc0f1846289d0c50f9fda71e76, entries=250, sequenceid=14, filesize=16.3 K 2024-11-16T10:23:54,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/1fbae99c4d214a67be37be4105945ca5 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/1fbae99c4d214a67be37be4105945ca5 2024-11-16T10:23:54,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/1fbae99c4d214a67be37be4105945ca5, entries=150, sequenceid=14, filesize=11.7 K 2024-11-16T10:23:54,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/61216607784a47a5ace23c975309c00b as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/61216607784a47a5ace23c975309c00b 2024-11-16T10:23:54,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T10:23:54,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/61216607784a47a5ace23c975309c00b, entries=150, sequenceid=14, filesize=11.7 K 2024-11-16T10:23:54,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:54,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:23:54,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:54,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:54,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:54,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for e63c645d595c80b0e763eedbebd195b2 in 479ms, sequenceid=14, compaction requested=false 2024-11-16T10:23:54,517 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-16T10:23:54,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:23:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:23:54,544 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:23:54,544 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e63c645d595c80b0e763eedbebd195b2 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T10:23:54,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/440807c8317f49a582f384d8bc42dc09 is 50, key is test_row_0/A:col10/1731752634539/Put/seqid=0 2024-11-16T10:23:54,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731752694569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:54,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:54,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T10:23:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37652 deadline: 1731752694567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752694573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:54,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:54,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:54,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752694577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:54,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:54,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752694582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:54,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:54,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741840_1016 (size=16681) 2024-11-16T10:23:54,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T10:23:54,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:54,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:23:54,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:54,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:54,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:54,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:54,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731752694697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:54,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37652 deadline: 1731752694698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:54,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:54,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752694698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:54,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:54,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752694699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:54,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:54,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:54,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752694700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:54,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:54,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T10:23:54,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:54,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:23:54,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:54,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:54,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:54,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752694917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:54,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37652 deadline: 1731752694917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:54,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731752694920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:54,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:54,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752694920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:54,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752694920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:54,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:54,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:54,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:54,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T10:23:54,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:54,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:23:54,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:54,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:54,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:55,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/440807c8317f49a582f384d8bc42dc09 2024-11-16T10:23:55,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/dea9f6ac4edb4d2bb93580ba120a5b9f is 50, key is test_row_0/B:col10/1731752634539/Put/seqid=0 2024-11-16T10:23:55,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741841_1017 (size=12001) 2024-11-16T10:23:55,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/dea9f6ac4edb4d2bb93580ba120a5b9f 2024-11-16T10:23:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T10:23:55,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/ec523327315e4b1ba7afdc359e77291d is 50, key is test_row_0/C:col10/1731752634539/Put/seqid=0 2024-11-16T10:23:55,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T10:23:55,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:55,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:23:55,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:55,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:55,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:55,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741842_1018 (size=12001) 2024-11-16T10:23:55,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/ec523327315e4b1ba7afdc359e77291d 2024-11-16T10:23:55,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/440807c8317f49a582f384d8bc42dc09 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/440807c8317f49a582f384d8bc42dc09 2024-11-16T10:23:55,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/440807c8317f49a582f384d8bc42dc09, entries=250, sequenceid=39, filesize=16.3 K 2024-11-16T10:23:55,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/dea9f6ac4edb4d2bb93580ba120a5b9f as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/dea9f6ac4edb4d2bb93580ba120a5b9f 2024-11-16T10:23:55,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/dea9f6ac4edb4d2bb93580ba120a5b9f, entries=150, sequenceid=39, filesize=11.7 K 2024-11-16T10:23:55,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/ec523327315e4b1ba7afdc359e77291d as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/ec523327315e4b1ba7afdc359e77291d 2024-11-16T10:23:55,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:55,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752695228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:55,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:55,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:55,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:55,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37652 deadline: 1731752695228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:55,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731752695228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:55,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:55,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:55,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752695237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:55,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:55,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:55,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752695238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:55,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:55,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/ec523327315e4b1ba7afdc359e77291d, entries=150, sequenceid=39, filesize=11.7 K 2024-11-16T10:23:55,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for e63c645d595c80b0e763eedbebd195b2 in 720ms, sequenceid=39, compaction requested=false 2024-11-16T10:23:55,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:23:55,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T10:23:55,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:55,297 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing e63c645d595c80b0e763eedbebd195b2 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T10:23:55,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/93debc24c7db46ec87d0bb05f3d1bb77 is 50, key is test_row_0/A:col10/1731752634576/Put/seqid=0 2024-11-16T10:23:55,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741843_1019 (size=12001) 2024-11-16T10:23:55,349 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/93debc24c7db46ec87d0bb05f3d1bb77 2024-11-16T10:23:55,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/afe9112f2756453caa08e5201563daf2 is 50, key is test_row_0/B:col10/1731752634576/Put/seqid=0 2024-11-16T10:23:55,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741844_1020 (size=12001) 2024-11-16T10:23:55,436 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/afe9112f2756453caa08e5201563daf2 2024-11-16T10:23:55,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/2fe18e4c7ffe4bc7971554dbdf144b88 is 50, key is test_row_0/C:col10/1731752634576/Put/seqid=0 2024-11-16T10:23:55,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741845_1021 (size=12001) 2024-11-16T10:23:55,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:23:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:23:55,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:55,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731752695811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37652 deadline: 1731752695810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:55,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:55,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:55,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752695816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:55,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:55,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752695819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:55,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:55,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:55,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752695821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:55,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:55,917 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/2fe18e4c7ffe4bc7971554dbdf144b88 2024-11-16T10:23:55,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731752695928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:55,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752695929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:55,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:55,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37652 deadline: 1731752695932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:55,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752695938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:55,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/93debc24c7db46ec87d0bb05f3d1bb77 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/93debc24c7db46ec87d0bb05f3d1bb77 2024-11-16T10:23:55,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752695938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:55,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:55,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:55,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:55,957 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/93debc24c7db46ec87d0bb05f3d1bb77, entries=150, sequenceid=50, filesize=11.7 K 2024-11-16T10:23:55,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/afe9112f2756453caa08e5201563daf2 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/afe9112f2756453caa08e5201563daf2 2024-11-16T10:23:55,973 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/afe9112f2756453caa08e5201563daf2, entries=150, sequenceid=50, filesize=11.7 K 2024-11-16T10:23:55,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/2fe18e4c7ffe4bc7971554dbdf144b88 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/2fe18e4c7ffe4bc7971554dbdf144b88 2024-11-16T10:23:55,994 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/2fe18e4c7ffe4bc7971554dbdf144b88, entries=150, sequenceid=50, filesize=11.7 K 2024-11-16T10:23:55,996 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e63c645d595c80b0e763eedbebd195b2 in 700ms, sequenceid=50, compaction requested=true 2024-11-16T10:23:55,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:23:55,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:55,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-16T10:23:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-16T10:23:56,006 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-16T10:23:56,006 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0310 sec 2024-11-16T10:23:56,011 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.0590 sec 2024-11-16T10:23:56,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T10:23:56,117 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:23:56,118 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:23:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-16T10:23:56,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T10:23:56,122 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:23:56,125 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:23:56,125 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:23:56,146 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:23:56,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e63c645d595c80b0e763eedbebd195b2 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-16T10:23:56,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:23:56,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/f10ee9814a4643cdb0c6c8b563fd214e is 50, key is test_row_0/A:col10/1731752636147/Put/seqid=0 2024-11-16T10:23:56,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741846_1022 (size=14341) 2024-11-16T10:23:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T10:23:56,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731752696156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:56,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:56,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752696184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:56,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:56,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:56,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752696228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:56,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:56,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:56,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752696230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:56,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:56,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37652 deadline: 1731752696229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:56,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:56,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T10:23:56,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:56,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:23:56,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:56,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:56,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:56,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:56,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731752696338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:56,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:56,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752696339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:56,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:56,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:56,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752696346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:56,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:56,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:56,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752696357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:56,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:56,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37652 deadline: 1731752696357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:56,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:56,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T10:23:56,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:56,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:23:56,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:56,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:56,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T10:23:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:56,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:56,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731752696547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:56,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:56,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752696558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:56,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:56,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:56,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752696558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:56,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:56,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:56,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752696568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:56,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:56,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37652 deadline: 1731752696568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:56,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/f10ee9814a4643cdb0c6c8b563fd214e 2024-11-16T10:23:56,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:56,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T10:23:56,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:56,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:23:56,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:56,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:56,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:56,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/d61e38faad0d437dbc19603f58291eed is 50, key is test_row_0/B:col10/1731752636147/Put/seqid=0 2024-11-16T10:23:56,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741847_1023 (size=12001) 2024-11-16T10:23:56,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/d61e38faad0d437dbc19603f58291eed 2024-11-16T10:23:56,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/3900fc6a2bc94519961def6b77ce1508 is 50, key is test_row_0/C:col10/1731752636147/Put/seqid=0 2024-11-16T10:23:56,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T10:23:56,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T10:23:56,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:56,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:23:56,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:56,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:56,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:56,764 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T10:23:56,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741848_1024 (size=12001) 2024-11-16T10:23:56,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/3900fc6a2bc94519961def6b77ce1508 2024-11-16T10:23:56,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/f10ee9814a4643cdb0c6c8b563fd214e as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/f10ee9814a4643cdb0c6c8b563fd214e 2024-11-16T10:23:56,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/f10ee9814a4643cdb0c6c8b563fd214e, entries=200, sequenceid=77, filesize=14.0 K 2024-11-16T10:23:56,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/d61e38faad0d437dbc19603f58291eed as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/d61e38faad0d437dbc19603f58291eed 2024-11-16T10:23:56,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:56,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731752696856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:56,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:56,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:56,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752696867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:56,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:56,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:56,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/d61e38faad0d437dbc19603f58291eed, entries=150, sequenceid=77, filesize=11.7 K 2024-11-16T10:23:56,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752696867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:56,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:56,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/3900fc6a2bc94519961def6b77ce1508 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/3900fc6a2bc94519961def6b77ce1508 2024-11-16T10:23:56,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:56,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37652 deadline: 1731752696892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:56,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:56,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:56,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/3900fc6a2bc94519961def6b77ce1508, entries=150, sequenceid=77, filesize=11.7 K 2024-11-16T10:23:56,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for e63c645d595c80b0e763eedbebd195b2 in 758ms, sequenceid=77, compaction requested=true 2024-11-16T10:23:56,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:23:56,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T10:23:56,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:56,909 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing e63c645d595c80b0e763eedbebd195b2 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-16T10:23:56,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/1365ef95725f4f6c920adbe1d813461a is 50, key is test_row_0/A:col10/1731752636188/Put/seqid=0 2024-11-16T10:23:56,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e63c645d595c80b0e763eedbebd195b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:23:56,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:23:56,931 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:23:56,940 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:23:56,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e63c645d595c80b0e763eedbebd195b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:23:56,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:23:56,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e63c645d595c80b0e763eedbebd195b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:23:56,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:23:56,946 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 59704 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:23:56,946 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:23:56,948 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): e63c645d595c80b0e763eedbebd195b2/B is initiating minor compaction (all files) 2024-11-16T10:23:56,948 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): e63c645d595c80b0e763eedbebd195b2/A is initiating minor compaction (all files) 2024-11-16T10:23:56,949 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e63c645d595c80b0e763eedbebd195b2/A in TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:56,949 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e63c645d595c80b0e763eedbebd195b2/B in TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:56,949 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/1fbae99c4d214a67be37be4105945ca5, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/dea9f6ac4edb4d2bb93580ba120a5b9f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/afe9112f2756453caa08e5201563daf2, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/d61e38faad0d437dbc19603f58291eed] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp, totalSize=46.9 K 2024-11-16T10:23:56,949 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/0b5b99bc0f1846289d0c50f9fda71e76, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/440807c8317f49a582f384d8bc42dc09, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/93debc24c7db46ec87d0bb05f3d1bb77, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/f10ee9814a4643cdb0c6c8b563fd214e] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp, totalSize=58.3 K 2024-11-16T10:23:56,951 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b5b99bc0f1846289d0c50f9fda71e76, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731752633958 2024-11-16T10:23:56,951 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fbae99c4d214a67be37be4105945ca5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731752633958 2024-11-16T10:23:56,951 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-16T10:23:56,953 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 440807c8317f49a582f384d8bc42dc09, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731752634174 2024-11-16T10:23:56,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:23:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:23:56,954 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting dea9f6ac4edb4d2bb93580ba120a5b9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731752634187 2024-11-16T10:23:56,955 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93debc24c7db46ec87d0bb05f3d1bb77, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731752634564 2024-11-16T10:23:56,955 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting afe9112f2756453caa08e5201563daf2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731752634564 2024-11-16T10:23:56,956 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting d61e38faad0d437dbc19603f58291eed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731752635812 2024-11-16T10:23:56,962 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting f10ee9814a4643cdb0c6c8b563fd214e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731752635812 2024-11-16T10:23:56,968 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-16T10:23:56,969 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-16T10:23:56,971 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T10:23:56,971 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-16T10:23:56,972 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-16T10:23:56,972 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-16T10:23:56,973 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T10:23:56,973 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-16T10:23:56,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741849_1025 (size=12001) 2024-11-16T10:23:57,036 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e63c645d595c80b0e763eedbebd195b2#A#compaction#13 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:23:57,036 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e63c645d595c80b0e763eedbebd195b2#B#compaction#14 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:23:57,037 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/32cc2adb0fed4323afa72e7f59543660 is 50, key is test_row_0/A:col10/1731752636147/Put/seqid=0 2024-11-16T10:23:57,037 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/048c9a6eb318480d937be3e5fc644622 is 50, key is test_row_0/B:col10/1731752636147/Put/seqid=0 2024-11-16T10:23:57,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741850_1026 (size=12139) 2024-11-16T10:23:57,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741851_1027 (size=12139) 2024-11-16T10:23:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T10:23:57,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752697305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:57,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731752697367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:57,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:57,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:57,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:57,389 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/1365ef95725f4f6c920adbe1d813461a 2024-11-16T10:23:57,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752697396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:57,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:57,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:57,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:57,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752697397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:57,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:57,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:57,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:57,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37652 deadline: 1731752697407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:57,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:57,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:57,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:57,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752697416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:57,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:57,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:57,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:57,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/a63b4374a72c42c6a66eab9c9c3986f3 is 50, key is test_row_0/B:col10/1731752636188/Put/seqid=0 2024-11-16T10:23:57,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741852_1028 (size=12001) 2024-11-16T10:23:57,458 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/a63b4374a72c42c6a66eab9c9c3986f3 2024-11-16T10:23:57,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/52362e1d518b47ca867a046d695e865f is 50, key is test_row_0/C:col10/1731752636188/Put/seqid=0 2024-11-16T10:23:57,504 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/32cc2adb0fed4323afa72e7f59543660 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/32cc2adb0fed4323afa72e7f59543660 2024-11-16T10:23:57,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741853_1029 (size=12001) 2024-11-16T10:23:57,518 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/52362e1d518b47ca867a046d695e865f 2024-11-16T10:23:57,534 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/048c9a6eb318480d937be3e5fc644622 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/048c9a6eb318480d937be3e5fc644622 2024-11-16T10:23:57,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/1365ef95725f4f6c920adbe1d813461a as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/1365ef95725f4f6c920adbe1d813461a 2024-11-16T10:23:57,544 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e63c645d595c80b0e763eedbebd195b2/A of e63c645d595c80b0e763eedbebd195b2 into 32cc2adb0fed4323afa72e7f59543660(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:23:57,544 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:23:57,548 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., storeName=e63c645d595c80b0e763eedbebd195b2/A, priority=12, startTime=1731752636906; duration=0sec 2024-11-16T10:23:57,549 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:23:57,549 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e63c645d595c80b0e763eedbebd195b2:A 2024-11-16T10:23:57,549 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:23:57,552 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/1365ef95725f4f6c920adbe1d813461a, entries=150, sequenceid=86, filesize=11.7 K 2024-11-16T10:23:57,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/a63b4374a72c42c6a66eab9c9c3986f3 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/a63b4374a72c42c6a66eab9c9c3986f3 2024-11-16T10:23:57,558 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e63c645d595c80b0e763eedbebd195b2/B of e63c645d595c80b0e763eedbebd195b2 into 048c9a6eb318480d937be3e5fc644622(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:23:57,558 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:23:57,558 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., storeName=e63c645d595c80b0e763eedbebd195b2/B, priority=12, startTime=1731752636928; duration=0sec 2024-11-16T10:23:57,558 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:23:57,558 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e63c645d595c80b0e763eedbebd195b2:B 2024-11-16T10:23:57,569 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:23:57,569 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): e63c645d595c80b0e763eedbebd195b2/C is initiating minor compaction (all files) 2024-11-16T10:23:57,569 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e63c645d595c80b0e763eedbebd195b2/C in TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:57,569 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/61216607784a47a5ace23c975309c00b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/ec523327315e4b1ba7afdc359e77291d, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/2fe18e4c7ffe4bc7971554dbdf144b88, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/3900fc6a2bc94519961def6b77ce1508] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp, totalSize=46.9 K 2024-11-16T10:23:57,571 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61216607784a47a5ace23c975309c00b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731752633958 2024-11-16T10:23:57,572 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec523327315e4b1ba7afdc359e77291d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731752634187 2024-11-16T10:23:57,577 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fe18e4c7ffe4bc7971554dbdf144b88, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731752634564 2024-11-16T10:23:57,578 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3900fc6a2bc94519961def6b77ce1508, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731752635812 2024-11-16T10:23:57,585 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/a63b4374a72c42c6a66eab9c9c3986f3, entries=150, sequenceid=86, filesize=11.7 K 2024-11-16T10:23:57,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/52362e1d518b47ca867a046d695e865f as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/52362e1d518b47ca867a046d695e865f 2024-11-16T10:23:57,613 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/52362e1d518b47ca867a046d695e865f, entries=150, sequenceid=86, filesize=11.7 K 2024-11-16T10:23:57,616 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for e63c645d595c80b0e763eedbebd195b2 in 708ms, sequenceid=86, compaction requested=false 2024-11-16T10:23:57,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:23:57,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:57,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-16T10:23:57,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-16T10:23:57,622 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e63c645d595c80b0e763eedbebd195b2#C#compaction#17 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:23:57,623 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/4a0a8da7b27047b9aef5ccbffa56923c is 50, key is test_row_0/C:col10/1731752636147/Put/seqid=0 2024-11-16T10:23:57,625 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-16T10:23:57,626 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4960 sec 2024-11-16T10:23:57,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741854_1030 (size=12139) 2024-11-16T10:23:57,631 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.5100 sec 2024-11-16T10:23:57,638 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:23:57,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e63c645d595c80b0e763eedbebd195b2 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-16T10:23:57,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:23:57,645 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/4a0a8da7b27047b9aef5ccbffa56923c as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/4a0a8da7b27047b9aef5ccbffa56923c 2024-11-16T10:23:57,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/1419b79484d44bfb94307530ab54b9eb is 50, key is test_row_0/A:col10/1731752637301/Put/seqid=0 2024-11-16T10:23:57,660 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e63c645d595c80b0e763eedbebd195b2/C of e63c645d595c80b0e763eedbebd195b2 into 4a0a8da7b27047b9aef5ccbffa56923c(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:23:57,662 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:23:57,662 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., storeName=e63c645d595c80b0e763eedbebd195b2/C, priority=12, startTime=1731752636941; duration=0sec 2024-11-16T10:23:57,662 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:23:57,663 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e63c645d595c80b0e763eedbebd195b2:C 2024-11-16T10:23:57,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741855_1031 (size=14341) 2024-11-16T10:23:57,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/1419b79484d44bfb94307530ab54b9eb 2024-11-16T10:23:57,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:57,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752697701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:57,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:57,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:57,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:57,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/0b10967bf22f41dbb8c677d6f67fd5c1 is 50, key is test_row_0/B:col10/1731752637301/Put/seqid=0 2024-11-16T10:23:57,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741856_1032 (size=12001) 2024-11-16T10:23:57,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/0b10967bf22f41dbb8c677d6f67fd5c1 2024-11-16T10:23:57,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/d2b144d39fd940b59548b4c562cb0ce8 is 50, key is test_row_0/C:col10/1731752637301/Put/seqid=0 2024-11-16T10:23:57,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:57,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752697816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:57,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:57,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:57,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:57,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741857_1033 (size=12001) 2024-11-16T10:23:57,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/d2b144d39fd940b59548b4c562cb0ce8 2024-11-16T10:23:57,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/1419b79484d44bfb94307530ab54b9eb as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/1419b79484d44bfb94307530ab54b9eb 2024-11-16T10:23:57,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/1419b79484d44bfb94307530ab54b9eb, entries=200, sequenceid=116, filesize=14.0 K 2024-11-16T10:23:57,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/0b10967bf22f41dbb8c677d6f67fd5c1 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/0b10967bf22f41dbb8c677d6f67fd5c1 2024-11-16T10:23:57,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/0b10967bf22f41dbb8c677d6f67fd5c1, entries=150, sequenceid=116, filesize=11.7 K 2024-11-16T10:23:57,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/d2b144d39fd940b59548b4c562cb0ce8 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/d2b144d39fd940b59548b4c562cb0ce8 2024-11-16T10:23:57,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/d2b144d39fd940b59548b4c562cb0ce8, entries=150, sequenceid=116, filesize=11.7 K 2024-11-16T10:23:57,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for e63c645d595c80b0e763eedbebd195b2 in 280ms, sequenceid=116, compaction requested=true 2024-11-16T10:23:57,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:23:57,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e63c645d595c80b0e763eedbebd195b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:23:57,919 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:23:57,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:23:57,919 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:23:57,922 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:23:57,922 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:23:57,922 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): e63c645d595c80b0e763eedbebd195b2/A is initiating minor compaction (all files) 2024-11-16T10:23:57,922 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): e63c645d595c80b0e763eedbebd195b2/B is initiating minor compaction (all files) 2024-11-16T10:23:57,922 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e63c645d595c80b0e763eedbebd195b2/A in TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:57,922 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e63c645d595c80b0e763eedbebd195b2/B in TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:57,922 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/32cc2adb0fed4323afa72e7f59543660, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/1365ef95725f4f6c920adbe1d813461a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/1419b79484d44bfb94307530ab54b9eb] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp, totalSize=37.6 K 2024-11-16T10:23:57,922 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/048c9a6eb318480d937be3e5fc644622, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/a63b4374a72c42c6a66eab9c9c3986f3, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/0b10967bf22f41dbb8c677d6f67fd5c1] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp, totalSize=35.3 K 2024-11-16T10:23:57,924 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32cc2adb0fed4323afa72e7f59543660, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731752635812 2024-11-16T10:23:57,924 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 048c9a6eb318480d937be3e5fc644622, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731752635812 2024-11-16T10:23:57,926 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1365ef95725f4f6c920adbe1d813461a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731752636156 2024-11-16T10:23:57,926 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting a63b4374a72c42c6a66eab9c9c3986f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731752636156 2024-11-16T10:23:57,927 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1419b79484d44bfb94307530ab54b9eb, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731752637230 2024-11-16T10:23:57,928 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b10967bf22f41dbb8c677d6f67fd5c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731752637261 2024-11-16T10:23:57,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e63c645d595c80b0e763eedbebd195b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:23:57,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:23:57,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e63c645d595c80b0e763eedbebd195b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:23:57,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:23:57,948 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e63c645d595c80b0e763eedbebd195b2#A#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:23:57,949 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/088658cd19344f39b9bd111e05e964ba is 50, key is test_row_0/A:col10/1731752637301/Put/seqid=0 2024-11-16T10:23:57,951 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e63c645d595c80b0e763eedbebd195b2#B#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:23:57,952 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/eae8d4a34c6b40c8b77a2180c74327fd is 50, key is test_row_0/B:col10/1731752637301/Put/seqid=0 2024-11-16T10:23:57,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741858_1034 (size=12241) 2024-11-16T10:23:57,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741859_1035 (size=12241) 2024-11-16T10:23:57,987 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/088658cd19344f39b9bd111e05e964ba as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/088658cd19344f39b9bd111e05e964ba 2024-11-16T10:23:57,998 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/eae8d4a34c6b40c8b77a2180c74327fd as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/eae8d4a34c6b40c8b77a2180c74327fd 2024-11-16T10:23:58,004 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e63c645d595c80b0e763eedbebd195b2/A of e63c645d595c80b0e763eedbebd195b2 into 088658cd19344f39b9bd111e05e964ba(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:23:58,005 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:23:58,005 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., storeName=e63c645d595c80b0e763eedbebd195b2/A, priority=13, startTime=1731752637919; duration=0sec 2024-11-16T10:23:58,005 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:23:58,005 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e63c645d595c80b0e763eedbebd195b2:A 2024-11-16T10:23:58,005 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:23:58,010 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:23:58,010 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): e63c645d595c80b0e763eedbebd195b2/C is initiating minor compaction (all files) 2024-11-16T10:23:58,010 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e63c645d595c80b0e763eedbebd195b2/C in TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:58,011 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/4a0a8da7b27047b9aef5ccbffa56923c, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/52362e1d518b47ca867a046d695e865f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/d2b144d39fd940b59548b4c562cb0ce8] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp, totalSize=35.3 K 2024-11-16T10:23:58,012 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a0a8da7b27047b9aef5ccbffa56923c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731752635812 2024-11-16T10:23:58,013 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52362e1d518b47ca867a046d695e865f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731752636156 2024-11-16T10:23:58,014 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2b144d39fd940b59548b4c562cb0ce8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731752637261 2024-11-16T10:23:58,019 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e63c645d595c80b0e763eedbebd195b2/B of e63c645d595c80b0e763eedbebd195b2 into eae8d4a34c6b40c8b77a2180c74327fd(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:23:58,019 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:23:58,019 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., storeName=e63c645d595c80b0e763eedbebd195b2/B, priority=13, startTime=1731752637919; duration=0sec 2024-11-16T10:23:58,020 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:23:58,020 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e63c645d595c80b0e763eedbebd195b2:B 2024-11-16T10:23:58,039 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e63c645d595c80b0e763eedbebd195b2#C#compaction#23 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:23:58,039 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/b86ca6c550ef468885aacce3b99995de is 50, key is test_row_0/C:col10/1731752637301/Put/seqid=0 2024-11-16T10:23:58,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:23:58,045 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:23:58,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e63c645d595c80b0e763eedbebd195b2 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T10:23:58,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/d39f96f86e4d4cb0b780c5c00e91ac03 is 50, key is test_row_0/A:col10/1731752638037/Put/seqid=0 2024-11-16T10:23:58,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741860_1036 (size=12241) 2024-11-16T10:23:58,090 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/b86ca6c550ef468885aacce3b99995de as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/b86ca6c550ef468885aacce3b99995de 2024-11-16T10:23:58,108 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e63c645d595c80b0e763eedbebd195b2/C of e63c645d595c80b0e763eedbebd195b2 into b86ca6c550ef468885aacce3b99995de(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:23:58,108 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:23:58,108 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., storeName=e63c645d595c80b0e763eedbebd195b2/C, priority=13, startTime=1731752637939; duration=0sec 2024-11-16T10:23:58,108 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:23:58,108 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e63c645d595c80b0e763eedbebd195b2:C 2024-11-16T10:23:58,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741861_1037 (size=14441) 2024-11-16T10:23:58,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:58,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752698245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:58,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:58,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T10:23:58,267 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:23:58,267 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:23:58,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-16T10:23:58,272 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:23:58,274 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:23:58,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:23:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T10:23:58,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:58,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752698356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:58,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:58,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:58,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T10:23:58,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:58,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731752698387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:58,393 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_0' in e63c645d595c80b0e763eedbebd195b2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:23:58,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:58,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:58,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:58,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752698416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:58,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:58,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752698417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:58,419 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_1' in e63c645d595c80b0e763eedbebd195b2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:23:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:58,421 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_0' in e63c645d595c80b0e763eedbebd195b2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:23:58,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:58,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:58,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:58,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:58,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37652 deadline: 1731752698427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:58,431 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_1' in e63c645d595c80b0e763eedbebd195b2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:23:58,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:58,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:58,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:58,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T10:23:58,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:58,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:23:58,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:58,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:58,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:58,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/d39f96f86e4d4cb0b780c5c00e91ac03 2024-11-16T10:23:58,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/754d806cb6124bd58fe066b4a1641a9e is 50, key is test_row_0/B:col10/1731752638037/Put/seqid=0 2024-11-16T10:23:58,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741862_1038 (size=12101) 2024-11-16T10:23:58,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:58,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752698566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:58,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:58,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:58,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:58,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/754d806cb6124bd58fe066b4a1641a9e 2024-11-16T10:23:58,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T10:23:58,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:58,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:23:58,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:58,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:58,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:58,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:58,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T10:23:58,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/43dcd99d485c4c62a1bb61f6ea92368e is 50, key is test_row_0/C:col10/1731752638037/Put/seqid=0 2024-11-16T10:23:58,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741863_1039 (size=12101) 2024-11-16T10:23:58,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T10:23:58,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:58,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:23:58,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:58,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:58,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:58,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:58,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:58,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752698876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:58,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:58,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:58,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:58,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T10:23:58,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:58,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:23:58,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:58,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:58,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:58,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:58,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T10:23:59,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/43dcd99d485c4c62a1bb61f6ea92368e 2024-11-16T10:23:59,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/d39f96f86e4d4cb0b780c5c00e91ac03 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/d39f96f86e4d4cb0b780c5c00e91ac03 2024-11-16T10:23:59,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T10:23:59,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:59,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:23:59,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:59,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:59,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:59,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:23:59,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/d39f96f86e4d4cb0b780c5c00e91ac03, entries=200, sequenceid=130, filesize=14.1 K 2024-11-16T10:23:59,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/754d806cb6124bd58fe066b4a1641a9e as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/754d806cb6124bd58fe066b4a1641a9e 2024-11-16T10:23:59,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/754d806cb6124bd58fe066b4a1641a9e, entries=150, sequenceid=130, filesize=11.8 K 2024-11-16T10:23:59,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/43dcd99d485c4c62a1bb61f6ea92368e as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/43dcd99d485c4c62a1bb61f6ea92368e 2024-11-16T10:23:59,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/43dcd99d485c4c62a1bb61f6ea92368e, entries=150, sequenceid=130, filesize=11.8 K 2024-11-16T10:23:59,112 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e63c645d595c80b0e763eedbebd195b2 in 1067ms, sequenceid=130, compaction requested=false 2024-11-16T10:23:59,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:23:59,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T10:23:59,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:59,210 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing e63c645d595c80b0e763eedbebd195b2 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T10:23:59,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/3dd8d545cc664fac893058bf4d884e65 is 50, key is test_row_0/A:col10/1731752638229/Put/seqid=0 2024-11-16T10:23:59,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741864_1040 (size=12151) 2024-11-16T10:23:59,272 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/3dd8d545cc664fac893058bf4d884e65 2024-11-16T10:23:59,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/d5995e3162194da5a53d4d049daefd0b is 50, key is test_row_0/B:col10/1731752638229/Put/seqid=0 2024-11-16T10:23:59,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741865_1041 (size=12151) 2024-11-16T10:23:59,312 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/d5995e3162194da5a53d4d049daefd0b 2024-11-16T10:23:59,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/e88f65af4c67464597dbf547776939e2 is 50, key is test_row_0/C:col10/1731752638229/Put/seqid=0 2024-11-16T10:23:59,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741866_1042 (size=12151) 2024-11-16T10:23:59,352 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/e88f65af4c67464597dbf547776939e2 2024-11-16T10:23:59,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/3dd8d545cc664fac893058bf4d884e65 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/3dd8d545cc664fac893058bf4d884e65 2024-11-16T10:23:59,382 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/3dd8d545cc664fac893058bf4d884e65, entries=150, sequenceid=156, filesize=11.9 K 2024-11-16T10:23:59,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/d5995e3162194da5a53d4d049daefd0b as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/d5995e3162194da5a53d4d049daefd0b 2024-11-16T10:23:59,396 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/d5995e3162194da5a53d4d049daefd0b, entries=150, sequenceid=156, filesize=11.9 K 2024-11-16T10:23:59,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:23:59,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:23:59,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/e88f65af4c67464597dbf547776939e2 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/e88f65af4c67464597dbf547776939e2 2024-11-16T10:23:59,409 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/e88f65af4c67464597dbf547776939e2, entries=150, sequenceid=156, filesize=11.9 K 2024-11-16T10:23:59,411 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=6.71 KB/6870 for e63c645d595c80b0e763eedbebd195b2 in 202ms, sequenceid=156, compaction requested=true 2024-11-16T10:23:59,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:23:59,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:59,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-16T10:23:59,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-16T10:23:59,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T10:23:59,418 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-16T10:23:59,419 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1400 sec 2024-11-16T10:23:59,423 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.1520 sec 2024-11-16T10:23:59,474 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:23:59,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e63c645d595c80b0e763eedbebd195b2 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T10:23:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:23:59,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/3b45821b5f7c435ebb897621fe1b2c36 is 50, key is test_row_0/A:col10/1731752639472/Put/seqid=0 2024-11-16T10:23:59,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741867_1043 (size=14541) 2024-11-16T10:23:59,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/3b45821b5f7c435ebb897621fe1b2c36 2024-11-16T10:23:59,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/ae3f0c266be04b539f148f1b0b3ae046 is 50, key is test_row_0/B:col10/1731752639472/Put/seqid=0 2024-11-16T10:23:59,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741868_1044 (size=12151) 2024-11-16T10:23:59,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/ae3f0c266be04b539f148f1b0b3ae046 2024-11-16T10:23:59,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/891faf2231e647dd8d3168e19083e87b is 50, key is test_row_0/C:col10/1731752639472/Put/seqid=0 2024-11-16T10:23:59,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741869_1045 (size=12151) 2024-11-16T10:23:59,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/891faf2231e647dd8d3168e19083e87b 2024-11-16T10:23:59,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/3b45821b5f7c435ebb897621fe1b2c36 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/3b45821b5f7c435ebb897621fe1b2c36 2024-11-16T10:23:59,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/3b45821b5f7c435ebb897621fe1b2c36, entries=200, sequenceid=167, filesize=14.2 K 2024-11-16T10:23:59,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/ae3f0c266be04b539f148f1b0b3ae046 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/ae3f0c266be04b539f148f1b0b3ae046 2024-11-16T10:23:59,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/ae3f0c266be04b539f148f1b0b3ae046, entries=150, sequenceid=167, filesize=11.9 K 2024-11-16T10:23:59,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/891faf2231e647dd8d3168e19083e87b as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/891faf2231e647dd8d3168e19083e87b 2024-11-16T10:23:59,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/891faf2231e647dd8d3168e19083e87b, entries=150, sequenceid=167, filesize=11.9 K 2024-11-16T10:23:59,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=127.47 KB/130530 for e63c645d595c80b0e763eedbebd195b2 in 240ms, sequenceid=167, compaction requested=true 2024-11-16T10:23:59,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:23:59,715 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:23:59,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e63c645d595c80b0e763eedbebd195b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:23:59,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:23:59,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e63c645d595c80b0e763eedbebd195b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:23:59,715 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:23:59,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:23:59,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e63c645d595c80b0e763eedbebd195b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:23:59,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:23:59,717 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53374 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:23:59,717 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): e63c645d595c80b0e763eedbebd195b2/A is initiating minor compaction (all files) 2024-11-16T10:23:59,718 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e63c645d595c80b0e763eedbebd195b2/A in TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:59,718 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/088658cd19344f39b9bd111e05e964ba, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/d39f96f86e4d4cb0b780c5c00e91ac03, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/3dd8d545cc664fac893058bf4d884e65, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/3b45821b5f7c435ebb897621fe1b2c36] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp, totalSize=52.1 K 2024-11-16T10:23:59,718 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:23:59,718 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): e63c645d595c80b0e763eedbebd195b2/B is initiating minor compaction (all files) 2024-11-16T10:23:59,718 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e63c645d595c80b0e763eedbebd195b2/B in TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:59,719 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/eae8d4a34c6b40c8b77a2180c74327fd, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/754d806cb6124bd58fe066b4a1641a9e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/d5995e3162194da5a53d4d049daefd0b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/ae3f0c266be04b539f148f1b0b3ae046] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp, totalSize=47.5 K 2024-11-16T10:23:59,719 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 088658cd19344f39b9bd111e05e964ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731752637261 2024-11-16T10:23:59,719 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting eae8d4a34c6b40c8b77a2180c74327fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731752637261 2024-11-16T10:23:59,720 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 754d806cb6124bd58fe066b4a1641a9e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731752638032 2024-11-16T10:23:59,720 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting d5995e3162194da5a53d4d049daefd0b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731752638211 2024-11-16T10:23:59,721 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting d39f96f86e4d4cb0b780c5c00e91ac03, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731752637664 2024-11-16T10:23:59,721 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting ae3f0c266be04b539f148f1b0b3ae046, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731752639455 2024-11-16T10:23:59,721 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dd8d545cc664fac893058bf4d884e65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731752638211 2024-11-16T10:23:59,722 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b45821b5f7c435ebb897621fe1b2c36, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731752639410 2024-11-16T10:23:59,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:23:59,752 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:23:59,752 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e63c645d595c80b0e763eedbebd195b2 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T10:23:59,754 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e63c645d595c80b0e763eedbebd195b2#B#compaction#33 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:23:59,755 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/62e726fdc5a14f709b23c2e9b3c627cc is 50, key is test_row_0/B:col10/1731752639472/Put/seqid=0 2024-11-16T10:23:59,764 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e63c645d595c80b0e763eedbebd195b2#A#compaction#34 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:23:59,765 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/d6a6e5b4d1c149e980a37062f3001187 is 50, key is test_row_0/A:col10/1731752639472/Put/seqid=0 2024-11-16T10:23:59,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/606ceb7832094eb68a9d828122834b43 is 50, key is test_row_0/A:col10/1731752639749/Put/seqid=0 2024-11-16T10:23:59,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741871_1047 (size=12527) 2024-11-16T10:23:59,818 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/d6a6e5b4d1c149e980a37062f3001187 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/d6a6e5b4d1c149e980a37062f3001187 2024-11-16T10:23:59,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741872_1048 (size=14541) 2024-11-16T10:23:59,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741870_1046 (size=12527) 2024-11-16T10:23:59,834 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e63c645d595c80b0e763eedbebd195b2/A of e63c645d595c80b0e763eedbebd195b2 into d6a6e5b4d1c149e980a37062f3001187(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:23:59,834 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:23:59,834 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., storeName=e63c645d595c80b0e763eedbebd195b2/A, priority=12, startTime=1731752639714; duration=0sec 2024-11-16T10:23:59,834 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:23:59,834 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e63c645d595c80b0e763eedbebd195b2:A 2024-11-16T10:23:59,834 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:23:59,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/606ceb7832094eb68a9d828122834b43 2024-11-16T10:23:59,842 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:23:59,843 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): e63c645d595c80b0e763eedbebd195b2/C is initiating minor compaction (all files) 2024-11-16T10:23:59,843 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e63c645d595c80b0e763eedbebd195b2/C in TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:23:59,843 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/b86ca6c550ef468885aacce3b99995de, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/43dcd99d485c4c62a1bb61f6ea92368e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/e88f65af4c67464597dbf547776939e2, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/891faf2231e647dd8d3168e19083e87b] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp, totalSize=47.5 K 2024-11-16T10:23:59,845 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting b86ca6c550ef468885aacce3b99995de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731752637261 2024-11-16T10:23:59,846 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43dcd99d485c4c62a1bb61f6ea92368e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731752638032 2024-11-16T10:23:59,847 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting e88f65af4c67464597dbf547776939e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731752638211 2024-11-16T10:23:59,849 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 891faf2231e647dd8d3168e19083e87b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731752639455 2024-11-16T10:23:59,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/4fdec96e905e4d0ca1d64d0b59923f14 is 50, key is test_row_0/B:col10/1731752639749/Put/seqid=0 2024-11-16T10:23:59,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752699866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:59,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:59,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:59,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:59,902 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e63c645d595c80b0e763eedbebd195b2#C#compaction#37 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:23:59,903 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/c04676ff5c8f4531b41149cea438d9ff is 50, key is test_row_0/C:col10/1731752639472/Put/seqid=0 2024-11-16T10:23:59,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741873_1049 (size=12151) 2024-11-16T10:23:59,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/4fdec96e905e4d0ca1d64d0b59923f14 2024-11-16T10:23:59,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/05bc4159766341a6b03c5678a7162f9a is 50, key is test_row_0/C:col10/1731752639749/Put/seqid=0 2024-11-16T10:23:59,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741874_1050 (size=12527) 2024-11-16T10:23:59,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:23:59,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752699977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:23:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:23:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:23:59,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741875_1051 (size=12151) 2024-11-16T10:23:59,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/05bc4159766341a6b03c5678a7162f9a 2024-11-16T10:23:59,986 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/c04676ff5c8f4531b41149cea438d9ff as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/c04676ff5c8f4531b41149cea438d9ff 2024-11-16T10:23:59,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/606ceb7832094eb68a9d828122834b43 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/606ceb7832094eb68a9d828122834b43 2024-11-16T10:24:00,003 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e63c645d595c80b0e763eedbebd195b2/C of e63c645d595c80b0e763eedbebd195b2 into c04676ff5c8f4531b41149cea438d9ff(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:00,003 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:00,003 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., storeName=e63c645d595c80b0e763eedbebd195b2/C, priority=12, startTime=1731752639715; duration=0sec 2024-11-16T10:24:00,003 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:00,004 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e63c645d595c80b0e763eedbebd195b2:C 2024-11-16T10:24:00,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/606ceb7832094eb68a9d828122834b43, entries=200, sequenceid=191, filesize=14.2 K 2024-11-16T10:24:00,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/4fdec96e905e4d0ca1d64d0b59923f14 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/4fdec96e905e4d0ca1d64d0b59923f14 2024-11-16T10:24:00,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/4fdec96e905e4d0ca1d64d0b59923f14, entries=150, sequenceid=191, filesize=11.9 K 2024-11-16T10:24:00,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/05bc4159766341a6b03c5678a7162f9a as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/05bc4159766341a6b03c5678a7162f9a 2024-11-16T10:24:00,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/05bc4159766341a6b03c5678a7162f9a, entries=150, sequenceid=191, filesize=11.9 K 2024-11-16T10:24:00,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for e63c645d595c80b0e763eedbebd195b2 in 283ms, sequenceid=191, compaction requested=false 2024-11-16T10:24:00,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:00,193 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:24:00,193 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e63c645d595c80b0e763eedbebd195b2 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T10:24:00,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:24:00,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/f8f13c7b51e34fc7ae49d5309355a28b is 50, key is test_row_0/A:col10/1731752639830/Put/seqid=0 2024-11-16T10:24:00,250 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/62e726fdc5a14f709b23c2e9b3c627cc as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/62e726fdc5a14f709b23c2e9b3c627cc 2024-11-16T10:24:00,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741876_1052 (size=12151) 2024-11-16T10:24:00,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/f8f13c7b51e34fc7ae49d5309355a28b 2024-11-16T10:24:00,265 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e63c645d595c80b0e763eedbebd195b2/B of e63c645d595c80b0e763eedbebd195b2 into 62e726fdc5a14f709b23c2e9b3c627cc(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:00,265 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:00,266 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., storeName=e63c645d595c80b0e763eedbebd195b2/B, priority=12, startTime=1731752639715; duration=0sec 2024-11-16T10:24:00,266 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:00,266 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e63c645d595c80b0e763eedbebd195b2:B 2024-11-16T10:24:00,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/e196b86eff8744b1995a29a812ae5069 is 50, key is test_row_0/B:col10/1731752639830/Put/seqid=0 2024-11-16T10:24:00,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741877_1053 (size=12151) 2024-11-16T10:24:00,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:00,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752700405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:00,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731752700407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:00,412 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_0' in e63c645d595c80b0e763eedbebd195b2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4257 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:24:00,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:00,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:00,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:00,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T10:24:00,428 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:24:00,429 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:24:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-16T10:24:00,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T10:24:00,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:00,433 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:24:00,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752700427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:00,434 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_1' in e63c645d595c80b0e763eedbebd195b2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:24:00,434 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:24:00,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:00,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:00,435 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:24:00,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:00,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752700447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:00,451 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_0' in e63c645d595c80b0e763eedbebd195b2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4267 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:24:00,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:00,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:00,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:00,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37652 deadline: 1731752700449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:00,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:00,456 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_1' in e63c645d595c80b0e763eedbebd195b2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:24:00,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:00,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:00,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:00,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:00,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752700516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:00,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:00,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:00,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:00,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T10:24:00,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T10:24:00,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:00,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:00,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:00,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:00,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:00,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:00,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752700727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:00,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:00,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:00,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/e196b86eff8744b1995a29a812ae5069 2024-11-16T10:24:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T10:24:00,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/84a52682c3704594be3112bbc398ffa2 is 50, key is test_row_0/C:col10/1731752639830/Put/seqid=0 2024-11-16T10:24:00,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T10:24:00,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:00,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:00,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:00,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:00,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:00,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:00,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741878_1054 (size=12151) 2024-11-16T10:24:00,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/84a52682c3704594be3112bbc398ffa2 2024-11-16T10:24:00,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/f8f13c7b51e34fc7ae49d5309355a28b as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/f8f13c7b51e34fc7ae49d5309355a28b 2024-11-16T10:24:00,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/f8f13c7b51e34fc7ae49d5309355a28b, entries=150, sequenceid=206, filesize=11.9 K 2024-11-16T10:24:00,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/e196b86eff8744b1995a29a812ae5069 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/e196b86eff8744b1995a29a812ae5069 2024-11-16T10:24:00,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/e196b86eff8744b1995a29a812ae5069, entries=150, sequenceid=206, filesize=11.9 K 2024-11-16T10:24:00,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/84a52682c3704594be3112bbc398ffa2 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/84a52682c3704594be3112bbc398ffa2 2024-11-16T10:24:00,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/84a52682c3704594be3112bbc398ffa2, entries=150, sequenceid=206, filesize=11.9 K 2024-11-16T10:24:00,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for e63c645d595c80b0e763eedbebd195b2 in 691ms, sequenceid=206, compaction requested=true 2024-11-16T10:24:00,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:00,885 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:24:00,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e63c645d595c80b0e763eedbebd195b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:24:00,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:00,885 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:24:00,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e63c645d595c80b0e763eedbebd195b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:24:00,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:00,887 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:24:00,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e63c645d595c80b0e763eedbebd195b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:24:00,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:24:00,887 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): e63c645d595c80b0e763eedbebd195b2/A is initiating minor compaction (all files) 2024-11-16T10:24:00,887 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e63c645d595c80b0e763eedbebd195b2/A in TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:00,888 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/d6a6e5b4d1c149e980a37062f3001187, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/606ceb7832094eb68a9d828122834b43, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/f8f13c7b51e34fc7ae49d5309355a28b] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp, totalSize=38.3 K 2024-11-16T10:24:00,888 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:24:00,889 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): e63c645d595c80b0e763eedbebd195b2/B is initiating minor compaction (all files) 2024-11-16T10:24:00,889 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e63c645d595c80b0e763eedbebd195b2/B in TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:00,889 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/62e726fdc5a14f709b23c2e9b3c627cc, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/4fdec96e905e4d0ca1d64d0b59923f14, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/e196b86eff8744b1995a29a812ae5069] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp, totalSize=36.0 K 2024-11-16T10:24:00,889 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6a6e5b4d1c149e980a37062f3001187, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731752639455 2024-11-16T10:24:00,890 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 62e726fdc5a14f709b23c2e9b3c627cc, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731752639455 2024-11-16T10:24:00,890 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fdec96e905e4d0ca1d64d0b59923f14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731752639677 2024-11-16T10:24:00,890 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 606ceb7832094eb68a9d828122834b43, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731752639677 2024-11-16T10:24:00,891 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting e196b86eff8744b1995a29a812ae5069, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731752639815 2024-11-16T10:24:00,891 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8f13c7b51e34fc7ae49d5309355a28b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731752639815 2024-11-16T10:24:00,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T10:24:00,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:00,908 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing e63c645d595c80b0e763eedbebd195b2 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T10:24:00,927 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e63c645d595c80b0e763eedbebd195b2#B#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:00,928 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/71743cbcebc447718c63d6ef15670b0a is 50, key is test_row_0/B:col10/1731752639830/Put/seqid=0 2024-11-16T10:24:00,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/e7b52a00abfc423dbce919b951aa299d is 50, key is test_row_0/A:col10/1731752640371/Put/seqid=0 2024-11-16T10:24:00,949 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e63c645d595c80b0e763eedbebd195b2#A#compaction#44 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:00,949 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/533350ee8d8b451e8e9a38ab9873979b is 50, key is test_row_0/A:col10/1731752639830/Put/seqid=0 2024-11-16T10:24:01,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741879_1055 (size=12629) 2024-11-16T10:24:01,021 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/71743cbcebc447718c63d6ef15670b0a as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/71743cbcebc447718c63d6ef15670b0a 2024-11-16T10:24:01,031 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e63c645d595c80b0e763eedbebd195b2/B of e63c645d595c80b0e763eedbebd195b2 into 71743cbcebc447718c63d6ef15670b0a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:01,032 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:01,032 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., storeName=e63c645d595c80b0e763eedbebd195b2/B, priority=13, startTime=1731752640885; duration=0sec 2024-11-16T10:24:01,032 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:24:01,032 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e63c645d595c80b0e763eedbebd195b2:B 2024-11-16T10:24:01,032 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:24:01,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741880_1056 (size=12151) 2024-11-16T10:24:01,036 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:24:01,036 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): e63c645d595c80b0e763eedbebd195b2/C is initiating minor compaction (all files) 2024-11-16T10:24:01,037 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e63c645d595c80b0e763eedbebd195b2/C in TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:01,037 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/c04676ff5c8f4531b41149cea438d9ff, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/05bc4159766341a6b03c5678a7162f9a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/84a52682c3704594be3112bbc398ffa2] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp, totalSize=36.0 K 2024-11-16T10:24:01,038 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting c04676ff5c8f4531b41149cea438d9ff, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731752639455 2024-11-16T10:24:01,039 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 05bc4159766341a6b03c5678a7162f9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731752639677 2024-11-16T10:24:01,039 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 84a52682c3704594be3112bbc398ffa2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731752639815 2024-11-16T10:24:01,047 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/e7b52a00abfc423dbce919b951aa299d 2024-11-16T10:24:01,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:01,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:24:01,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T10:24:01,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741881_1057 (size=12629) 2024-11-16T10:24:01,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/ff997d7e9f2b49089cf60aab618257f7 is 50, key is test_row_0/B:col10/1731752640371/Put/seqid=0 2024-11-16T10:24:01,067 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e63c645d595c80b0e763eedbebd195b2#C#compaction#46 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:01,068 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/64b023defa2a4f34b853e3e05050f26a is 50, key is test_row_0/C:col10/1731752639830/Put/seqid=0 2024-11-16T10:24:01,081 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/533350ee8d8b451e8e9a38ab9873979b as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/533350ee8d8b451e8e9a38ab9873979b 2024-11-16T10:24:01,095 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e63c645d595c80b0e763eedbebd195b2/A of e63c645d595c80b0e763eedbebd195b2 into 533350ee8d8b451e8e9a38ab9873979b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:01,095 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:01,096 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., storeName=e63c645d595c80b0e763eedbebd195b2/A, priority=13, startTime=1731752640884; duration=0sec 2024-11-16T10:24:01,096 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:01,096 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e63c645d595c80b0e763eedbebd195b2:A 2024-11-16T10:24:01,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741882_1058 (size=12151) 2024-11-16T10:24:01,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741883_1059 (size=12629) 2024-11-16T10:24:01,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:01,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752701151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:01,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752701256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:01,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:01,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:01,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:01,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752701466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:01,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:01,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:01,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:01,514 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/ff997d7e9f2b49089cf60aab618257f7 2024-11-16T10:24:01,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/e9c930d074374255a9b12516ab62fc8c is 50, key is test_row_0/C:col10/1731752640371/Put/seqid=0 2024-11-16T10:24:01,544 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/64b023defa2a4f34b853e3e05050f26a as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/64b023defa2a4f34b853e3e05050f26a 2024-11-16T10:24:01,562 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e63c645d595c80b0e763eedbebd195b2/C of e63c645d595c80b0e763eedbebd195b2 into 64b023defa2a4f34b853e3e05050f26a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:01,562 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:01,562 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., storeName=e63c645d595c80b0e763eedbebd195b2/C, priority=13, startTime=1731752640887; duration=0sec 2024-11-16T10:24:01,563 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:01,563 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e63c645d595c80b0e763eedbebd195b2:C 2024-11-16T10:24:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T10:24:01,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741884_1060 (size=12151) 2024-11-16T10:24:01,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:01,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752701777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:01,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:01,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:01,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:01,971 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/e9c930d074374255a9b12516ab62fc8c 2024-11-16T10:24:01,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/e7b52a00abfc423dbce919b951aa299d as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/e7b52a00abfc423dbce919b951aa299d 2024-11-16T10:24:01,989 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/e7b52a00abfc423dbce919b951aa299d, entries=150, sequenceid=230, filesize=11.9 K 2024-11-16T10:24:01,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/ff997d7e9f2b49089cf60aab618257f7 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/ff997d7e9f2b49089cf60aab618257f7 2024-11-16T10:24:02,010 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/ff997d7e9f2b49089cf60aab618257f7, entries=150, sequenceid=230, filesize=11.9 K 2024-11-16T10:24:02,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/e9c930d074374255a9b12516ab62fc8c as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/e9c930d074374255a9b12516ab62fc8c 2024-11-16T10:24:02,033 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/e9c930d074374255a9b12516ab62fc8c, entries=150, sequenceid=230, filesize=11.9 K 2024-11-16T10:24:02,035 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for e63c645d595c80b0e763eedbebd195b2 in 1127ms, sequenceid=230, compaction requested=false 2024-11-16T10:24:02,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:02,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:02,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-16T10:24:02,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-16T10:24:02,040 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-16T10:24:02,041 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6020 sec 2024-11-16T10:24:02,069 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.6120 sec 2024-11-16T10:24:02,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:24:02,291 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:24:02,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e63c645d595c80b0e763eedbebd195b2 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-16T10:24:02,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/3a7c5c1ea7784149b72131dd01ef3b38 is 50, key is test_row_0/A:col10/1731752641112/Put/seqid=0 2024-11-16T10:24:02,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741885_1061 (size=12151) 2024-11-16T10:24:02,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752702396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:02,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:02,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:02,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:02,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:02,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752702506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:02,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T10:24:02,577 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:24:02,577 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:24:02,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-16T10:24:02,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T10:24:02,579 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:24:02,580 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:24:02,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:24:02,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T10:24:02,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:02,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752702716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:02,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:02,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:02,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:02,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/3a7c5c1ea7784149b72131dd01ef3b38 2024-11-16T10:24:02,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T10:24:02,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:02,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:02,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:02,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:02,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:02,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:02,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/04cb683d56bb4b3aaea9609af1826d7b is 50, key is test_row_0/B:col10/1731752641112/Put/seqid=0 2024-11-16T10:24:02,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741886_1062 (size=12151) 2024-11-16T10:24:02,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/04cb683d56bb4b3aaea9609af1826d7b 2024-11-16T10:24:02,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/e12df7db30954c4a9b54b4210731588c is 50, key is test_row_0/C:col10/1731752641112/Put/seqid=0 2024-11-16T10:24:02,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741887_1063 (size=12151) 2024-11-16T10:24:02,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T10:24:02,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:02,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:02,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:02,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:02,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:02,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T10:24:03,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:03,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752703026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:03,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T10:24:03,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:03,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:03,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:03,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:03,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:03,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/e12df7db30954c4a9b54b4210731588c 2024-11-16T10:24:03,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/3a7c5c1ea7784149b72131dd01ef3b38 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/3a7c5c1ea7784149b72131dd01ef3b38 2024-11-16T10:24:03,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T10:24:03,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:03,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:03,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:03,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:03,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:03,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/3a7c5c1ea7784149b72131dd01ef3b38, entries=150, sequenceid=247, filesize=11.9 K 2024-11-16T10:24:03,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/04cb683d56bb4b3aaea9609af1826d7b as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/04cb683d56bb4b3aaea9609af1826d7b 2024-11-16T10:24:03,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T10:24:03,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/04cb683d56bb4b3aaea9609af1826d7b, entries=150, sequenceid=247, filesize=11.9 K 2024-11-16T10:24:03,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/e12df7db30954c4a9b54b4210731588c as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/e12df7db30954c4a9b54b4210731588c 2024-11-16T10:24:03,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/e12df7db30954c4a9b54b4210731588c, entries=150, sequenceid=247, filesize=11.9 K 2024-11-16T10:24:03,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for e63c645d595c80b0e763eedbebd195b2 in 931ms, sequenceid=247, compaction requested=true 2024-11-16T10:24:03,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:03,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e63c645d595c80b0e763eedbebd195b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:24:03,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:03,222 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:24:03,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e63c645d595c80b0e763eedbebd195b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:24:03,223 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:24:03,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:03,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e63c645d595c80b0e763eedbebd195b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:24:03,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:24:03,224 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:24:03,224 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): e63c645d595c80b0e763eedbebd195b2/A is initiating minor compaction (all files) 2024-11-16T10:24:03,224 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e63c645d595c80b0e763eedbebd195b2/A in TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:03,224 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/533350ee8d8b451e8e9a38ab9873979b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/e7b52a00abfc423dbce919b951aa299d, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/3a7c5c1ea7784149b72131dd01ef3b38] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp, totalSize=36.1 K 2024-11-16T10:24:03,224 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:24:03,224 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): e63c645d595c80b0e763eedbebd195b2/B is initiating minor compaction (all files) 2024-11-16T10:24:03,224 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 533350ee8d8b451e8e9a38ab9873979b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731752639815 2024-11-16T10:24:03,224 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e63c645d595c80b0e763eedbebd195b2/B in TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:03,225 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/71743cbcebc447718c63d6ef15670b0a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/ff997d7e9f2b49089cf60aab618257f7, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/04cb683d56bb4b3aaea9609af1826d7b] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp, totalSize=36.1 K 2024-11-16T10:24:03,225 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 71743cbcebc447718c63d6ef15670b0a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731752639815 2024-11-16T10:24:03,225 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7b52a00abfc423dbce919b951aa299d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731752640361 2024-11-16T10:24:03,226 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting ff997d7e9f2b49089cf60aab618257f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731752640361 2024-11-16T10:24:03,226 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a7c5c1ea7784149b72131dd01ef3b38, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731752641112 2024-11-16T10:24:03,227 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 04cb683d56bb4b3aaea9609af1826d7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731752641112 2024-11-16T10:24:03,251 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e63c645d595c80b0e763eedbebd195b2#A#compaction#51 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:03,251 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/cc349a464bdf423284cd4fda424feb0e is 50, key is test_row_0/A:col10/1731752641112/Put/seqid=0 2024-11-16T10:24:03,254 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e63c645d595c80b0e763eedbebd195b2#B#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:03,254 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/fff44d94818d4c00aa7e21d55a60f020 is 50, key is test_row_0/B:col10/1731752641112/Put/seqid=0 2024-11-16T10:24:03,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741888_1064 (size=12731) 2024-11-16T10:24:03,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741889_1065 (size=12731) 2024-11-16T10:24:03,284 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/fff44d94818d4c00aa7e21d55a60f020 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/fff44d94818d4c00aa7e21d55a60f020 2024-11-16T10:24:03,294 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e63c645d595c80b0e763eedbebd195b2/B of e63c645d595c80b0e763eedbebd195b2 into fff44d94818d4c00aa7e21d55a60f020(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:03,294 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:03,294 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., storeName=e63c645d595c80b0e763eedbebd195b2/B, priority=13, startTime=1731752643223; duration=0sec 2024-11-16T10:24:03,295 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:24:03,295 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e63c645d595c80b0e763eedbebd195b2:B 2024-11-16T10:24:03,295 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:24:03,298 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:24:03,298 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): e63c645d595c80b0e763eedbebd195b2/C is initiating minor compaction (all files) 2024-11-16T10:24:03,298 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e63c645d595c80b0e763eedbebd195b2/C in TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:03,298 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/64b023defa2a4f34b853e3e05050f26a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/e9c930d074374255a9b12516ab62fc8c, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/e12df7db30954c4a9b54b4210731588c] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp, totalSize=36.1 K 2024-11-16T10:24:03,299 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 64b023defa2a4f34b853e3e05050f26a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731752639815 2024-11-16T10:24:03,299 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting e9c930d074374255a9b12516ab62fc8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731752640361 2024-11-16T10:24:03,300 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting e12df7db30954c4a9b54b4210731588c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731752641112 2024-11-16T10:24:03,315 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e63c645d595c80b0e763eedbebd195b2#C#compaction#53 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:03,315 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/b96336fe9b224503b5fd5eb6103b7e15 is 50, key is test_row_0/C:col10/1731752641112/Put/seqid=0 2024-11-16T10:24:03,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741890_1066 (size=12731) 2024-11-16T10:24:03,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T10:24:03,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:03,351 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing e63c645d595c80b0e763eedbebd195b2 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-16T10:24:03,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/007c6c5f05184c2f82f7b4ad611f67f0 is 50, key is test_row_0/A:col10/1731752642383/Put/seqid=0 2024-11-16T10:24:03,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741891_1067 (size=12301) 2024-11-16T10:24:03,378 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/007c6c5f05184c2f82f7b4ad611f67f0 2024-11-16T10:24:03,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/057dd860bb4d4f758e4e130c7780b6a7 is 50, key is test_row_0/B:col10/1731752642383/Put/seqid=0 2024-11-16T10:24:03,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741892_1068 (size=12301) 2024-11-16T10:24:03,401 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/057dd860bb4d4f758e4e130c7780b6a7 2024-11-16T10:24:03,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/d7b089dc014849c7ad1d8c62de1f1111 is 50, key is test_row_0/C:col10/1731752642383/Put/seqid=0 2024-11-16T10:24:03,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741893_1069 (size=12301) 2024-11-16T10:24:03,419 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/d7b089dc014849c7ad1d8c62de1f1111 2024-11-16T10:24:03,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/007c6c5f05184c2f82f7b4ad611f67f0 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/007c6c5f05184c2f82f7b4ad611f67f0 2024-11-16T10:24:03,434 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/007c6c5f05184c2f82f7b4ad611f67f0, entries=150, sequenceid=270, filesize=12.0 K 2024-11-16T10:24:03,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/057dd860bb4d4f758e4e130c7780b6a7 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/057dd860bb4d4f758e4e130c7780b6a7 2024-11-16T10:24:03,440 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/057dd860bb4d4f758e4e130c7780b6a7, entries=150, sequenceid=270, filesize=12.0 K 2024-11-16T10:24:03,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/d7b089dc014849c7ad1d8c62de1f1111 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/d7b089dc014849c7ad1d8c62de1f1111 2024-11-16T10:24:03,451 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/d7b089dc014849c7ad1d8c62de1f1111, entries=150, sequenceid=270, filesize=12.0 K 2024-11-16T10:24:03,452 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for e63c645d595c80b0e763eedbebd195b2 in 102ms, sequenceid=270, compaction requested=false 2024-11-16T10:24:03,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:03,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:03,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-16T10:24:03,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-16T10:24:03,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-16T10:24:03,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 874 msec 2024-11-16T10:24:03,463 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 883 msec 2024-11-16T10:24:03,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:24:03,561 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:24:03,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e63c645d595c80b0e763eedbebd195b2 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T10:24:03,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/0fb5d05194a54cfc99c1144c30221d1b is 50, key is test_row_0/A:col10/1731752643555/Put/seqid=0 2024-11-16T10:24:03,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741894_1070 (size=14741) 2024-11-16T10:24:03,578 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/0fb5d05194a54cfc99c1144c30221d1b 2024-11-16T10:24:03,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/9397539034664aa4bbad996f0d14bd6c is 50, key is test_row_0/B:col10/1731752643555/Put/seqid=0 2024-11-16T10:24:03,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741895_1071 (size=12301) 2024-11-16T10:24:03,680 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/cc349a464bdf423284cd4fda424feb0e as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/cc349a464bdf423284cd4fda424feb0e 2024-11-16T10:24:03,686 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e63c645d595c80b0e763eedbebd195b2/A of e63c645d595c80b0e763eedbebd195b2 into cc349a464bdf423284cd4fda424feb0e(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:03,686 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:03,686 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., storeName=e63c645d595c80b0e763eedbebd195b2/A, priority=13, startTime=1731752643222; duration=0sec 2024-11-16T10:24:03,686 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:03,686 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e63c645d595c80b0e763eedbebd195b2:A 2024-11-16T10:24:03,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752703686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:03,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:03,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:03,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T10:24:03,716 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:24:03,716 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:24:03,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-16T10:24:03,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T10:24:03,718 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:24:03,719 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:24:03,719 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:24:03,738 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/b96336fe9b224503b5fd5eb6103b7e15 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/b96336fe9b224503b5fd5eb6103b7e15 2024-11-16T10:24:03,751 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e63c645d595c80b0e763eedbebd195b2/C of e63c645d595c80b0e763eedbebd195b2 into b96336fe9b224503b5fd5eb6103b7e15(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:03,751 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:03,751 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., storeName=e63c645d595c80b0e763eedbebd195b2/C, priority=13, startTime=1731752643223; duration=0sec 2024-11-16T10:24:03,751 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:03,751 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e63c645d595c80b0e763eedbebd195b2:C 2024-11-16T10:24:03,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:03,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752703797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:03,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:03,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:03,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:03,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T10:24:03,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T10:24:03,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:03,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:03,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:03,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:03,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:03,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:04,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:04,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752704006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:04,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:04,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:04,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:04,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/9397539034664aa4bbad996f0d14bd6c 2024-11-16T10:24:04,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T10:24:04,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:04,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:04,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:04,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:04,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:04,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:04,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T10:24:04,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/005bebc101734d0da2161fa29674a561 is 50, key is test_row_0/C:col10/1731752643555/Put/seqid=0 2024-11-16T10:24:04,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741896_1072 (size=12301) 2024-11-16T10:24:04,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T10:24:04,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:04,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:04,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:04,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:04,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:04,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:04,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:04,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752704316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:04,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:04,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:04,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:04,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T10:24:04,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:04,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:04,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:04,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:04,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:04,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:04,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T10:24:04,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:04,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731752704446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:04,447 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_0' in e63c645d595c80b0e763eedbebd195b2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8292 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:24:04,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:04,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:04,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:04,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/005bebc101734d0da2161fa29674a561 2024-11-16T10:24:04,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/0fb5d05194a54cfc99c1144c30221d1b as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/0fb5d05194a54cfc99c1144c30221d1b 2024-11-16T10:24:04,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/0fb5d05194a54cfc99c1144c30221d1b, entries=200, sequenceid=281, filesize=14.4 K 2024-11-16T10:24:04,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/9397539034664aa4bbad996f0d14bd6c as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/9397539034664aa4bbad996f0d14bd6c 2024-11-16T10:24:04,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/9397539034664aa4bbad996f0d14bd6c, entries=150, sequenceid=281, filesize=12.0 K 2024-11-16T10:24:04,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/005bebc101734d0da2161fa29674a561 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/005bebc101734d0da2161fa29674a561 2024-11-16T10:24:04,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:04,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37652 deadline: 1731752704466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:04,468 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_1' in e63c645d595c80b0e763eedbebd195b2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:24:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:04,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/005bebc101734d0da2161fa29674a561, entries=150, sequenceid=281, filesize=12.0 K 2024-11-16T10:24:04,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e63c645d595c80b0e763eedbebd195b2 in 914ms, sequenceid=281, compaction requested=true 2024-11-16T10:24:04,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:04,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e63c645d595c80b0e763eedbebd195b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:24:04,476 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:24:04,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:04,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e63c645d595c80b0e763eedbebd195b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:24:04,476 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:24:04,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:04,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e63c645d595c80b0e763eedbebd195b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:24:04,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:24:04,477 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:24:04,477 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:24:04,477 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): e63c645d595c80b0e763eedbebd195b2/A is initiating minor compaction (all files) 2024-11-16T10:24:04,477 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): e63c645d595c80b0e763eedbebd195b2/B is initiating minor compaction (all files) 2024-11-16T10:24:04,477 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e63c645d595c80b0e763eedbebd195b2/A in TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:04,477 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e63c645d595c80b0e763eedbebd195b2/B in TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:04,477 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/fff44d94818d4c00aa7e21d55a60f020, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/057dd860bb4d4f758e4e130c7780b6a7, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/9397539034664aa4bbad996f0d14bd6c] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp, totalSize=36.5 K 2024-11-16T10:24:04,477 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/cc349a464bdf423284cd4fda424feb0e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/007c6c5f05184c2f82f7b4ad611f67f0, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/0fb5d05194a54cfc99c1144c30221d1b] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp, totalSize=38.8 K 2024-11-16T10:24:04,478 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting fff44d94818d4c00aa7e21d55a60f020, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731752641112 2024-11-16T10:24:04,478 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc349a464bdf423284cd4fda424feb0e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731752641112 2024-11-16T10:24:04,478 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 057dd860bb4d4f758e4e130c7780b6a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731752642372 2024-11-16T10:24:04,479 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 007c6c5f05184c2f82f7b4ad611f67f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731752642372 2024-11-16T10:24:04,479 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 9397539034664aa4bbad996f0d14bd6c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731752643549 2024-11-16T10:24:04,480 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fb5d05194a54cfc99c1144c30221d1b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731752643540 2024-11-16T10:24:04,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:24:04,484 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:24:04,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e63c645d595c80b0e763eedbebd195b2 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-16T10:24:04,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T10:24:04,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:04,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:04,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:04,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:04,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:04,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:04,499 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e63c645d595c80b0e763eedbebd195b2#B#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:04,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/d17eab35c4f046508bece576327e7d5a is 50, key is test_row_0/A:col10/1731752644480/Put/seqid=0 2024-11-16T10:24:04,499 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e63c645d595c80b0e763eedbebd195b2#A#compaction#62 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:04,500 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/b8c4eff5c7344b5892e22817096a04c8 is 50, key is test_row_0/B:col10/1731752643555/Put/seqid=0 2024-11-16T10:24:04,500 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/a9387214362447ddbae78595a2d178cd is 50, key is test_row_0/A:col10/1731752643555/Put/seqid=0 2024-11-16T10:24:04,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:04,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752704504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:04,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:04,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752704506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:04,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:04,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:04,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:04,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741897_1073 (size=14741) 2024-11-16T10:24:04,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741898_1074 (size=12983) 2024-11-16T10:24:04,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741899_1075 (size=12983) 2024-11-16T10:24:04,516 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/a9387214362447ddbae78595a2d178cd as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/a9387214362447ddbae78595a2d178cd 2024-11-16T10:24:04,516 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/b8c4eff5c7344b5892e22817096a04c8 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/b8c4eff5c7344b5892e22817096a04c8 2024-11-16T10:24:04,524 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e63c645d595c80b0e763eedbebd195b2/B of e63c645d595c80b0e763eedbebd195b2 into b8c4eff5c7344b5892e22817096a04c8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:04,524 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e63c645d595c80b0e763eedbebd195b2/A of e63c645d595c80b0e763eedbebd195b2 into a9387214362447ddbae78595a2d178cd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:04,524 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:04,524 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:04,524 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., storeName=e63c645d595c80b0e763eedbebd195b2/B, priority=13, startTime=1731752644476; duration=0sec 2024-11-16T10:24:04,524 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., storeName=e63c645d595c80b0e763eedbebd195b2/A, priority=13, startTime=1731752644475; duration=0sec 2024-11-16T10:24:04,524 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:24:04,524 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e63c645d595c80b0e763eedbebd195b2:B 2024-11-16T10:24:04,524 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:24:04,524 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:04,525 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e63c645d595c80b0e763eedbebd195b2:A 2024-11-16T10:24:04,526 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:24:04,526 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): e63c645d595c80b0e763eedbebd195b2/C is initiating minor compaction (all files) 2024-11-16T10:24:04,526 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e63c645d595c80b0e763eedbebd195b2/C in TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:04,526 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/b96336fe9b224503b5fd5eb6103b7e15, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/d7b089dc014849c7ad1d8c62de1f1111, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/005bebc101734d0da2161fa29674a561] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp, totalSize=36.5 K 2024-11-16T10:24:04,527 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting b96336fe9b224503b5fd5eb6103b7e15, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731752641112 2024-11-16T10:24:04,527 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting d7b089dc014849c7ad1d8c62de1f1111, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731752642372 2024-11-16T10:24:04,527 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 005bebc101734d0da2161fa29674a561, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731752643549 2024-11-16T10:24:04,536 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e63c645d595c80b0e763eedbebd195b2#C#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:04,537 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/e46269ee704444c8b4b96be08091e572 is 50, key is test_row_0/C:col10/1731752643555/Put/seqid=0 2024-11-16T10:24:04,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741900_1076 (size=12983) 2024-11-16T10:24:04,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:04,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752704617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:04,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:04,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:04,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:04,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:04,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752704617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:04,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T10:24:04,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:04,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:04,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:04,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:04,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:04,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:04,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T10:24:04,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:04,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:04,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:04,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:04,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:04,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:04,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:04,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752704827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:04,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:04,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:04,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752704827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:04,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752704827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:04,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T10:24:04,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/d17eab35c4f046508bece576327e7d5a 2024-11-16T10:24:04,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/4567316d21df4e498e53add9e39861c9 is 50, key is test_row_0/B:col10/1731752644480/Put/seqid=0 2024-11-16T10:24:04,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741901_1077 (size=12301) 2024-11-16T10:24:04,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/4567316d21df4e498e53add9e39861c9 2024-11-16T10:24:04,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/a25f9ddc0a4c49ea849d4bb444516c19 is 50, key is test_row_0/C:col10/1731752644480/Put/seqid=0 2024-11-16T10:24:04,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741902_1078 (size=12301) 2024-11-16T10:24:04,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T10:24:04,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:04,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:04,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:04,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:04,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:04,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:04,961 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/e46269ee704444c8b4b96be08091e572 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/e46269ee704444c8b4b96be08091e572 2024-11-16T10:24:04,968 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e63c645d595c80b0e763eedbebd195b2/C of e63c645d595c80b0e763eedbebd195b2 into e46269ee704444c8b4b96be08091e572(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:04,968 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:04,968 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., storeName=e63c645d595c80b0e763eedbebd195b2/C, priority=13, startTime=1731752644476; duration=0sec 2024-11-16T10:24:04,968 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:04,968 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e63c645d595c80b0e763eedbebd195b2:C 2024-11-16T10:24:05,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T10:24:05,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:05,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:05,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:05,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:05,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:05,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:05,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:05,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752705137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:05,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:05,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752705137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:05,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:05,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:05,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:05,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T10:24:05,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:05,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:05,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:05,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:05,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:05,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:05,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/a25f9ddc0a4c49ea849d4bb444516c19 2024-11-16T10:24:05,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/d17eab35c4f046508bece576327e7d5a as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/d17eab35c4f046508bece576327e7d5a 2024-11-16T10:24:05,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/d17eab35c4f046508bece576327e7d5a, entries=200, sequenceid=311, filesize=14.4 K 2024-11-16T10:24:05,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/4567316d21df4e498e53add9e39861c9 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/4567316d21df4e498e53add9e39861c9 2024-11-16T10:24:05,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/4567316d21df4e498e53add9e39861c9, entries=150, sequenceid=311, filesize=12.0 K 2024-11-16T10:24:05,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/a25f9ddc0a4c49ea849d4bb444516c19 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/a25f9ddc0a4c49ea849d4bb444516c19 2024-11-16T10:24:05,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/a25f9ddc0a4c49ea849d4bb444516c19, entries=150, sequenceid=311, filesize=12.0 K 2024-11-16T10:24:05,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for e63c645d595c80b0e763eedbebd195b2 in 901ms, sequenceid=311, compaction requested=false 2024-11-16T10:24:05,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:05,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T10:24:05,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:05,431 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing e63c645d595c80b0e763eedbebd195b2 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-16T10:24:05,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/724507c405cc47f6abea8d56cf1ba0b0 is 50, key is test_row_0/A:col10/1731752644501/Put/seqid=0 2024-11-16T10:24:05,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741903_1079 (size=12301) 2024-11-16T10:24:05,470 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/724507c405cc47f6abea8d56cf1ba0b0 2024-11-16T10:24:05,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/ef634949411e4f0792ed60d685716beb is 50, key is test_row_0/B:col10/1731752644501/Put/seqid=0 2024-11-16T10:24:05,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741904_1080 (size=12301) 2024-11-16T10:24:05,529 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/ef634949411e4f0792ed60d685716beb 2024-11-16T10:24:05,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/4b158d8efdba4f1b950e63b36c0a3e6f is 50, key is test_row_0/C:col10/1731752644501/Put/seqid=0 2024-11-16T10:24:05,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741905_1081 (size=12301) 2024-11-16T10:24:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:24:05,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:05,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752705705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:05,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752705708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:05,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:05,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752705816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:05,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:05,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:05,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:05,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:05,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752705817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:05,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:05,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752705846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:05,847 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_2' in e63c645d595c80b0e763eedbebd195b2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:24:05,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:05,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:05,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:05,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T10:24:05,956 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/4b158d8efdba4f1b950e63b36c0a3e6f 2024-11-16T10:24:05,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/724507c405cc47f6abea8d56cf1ba0b0 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/724507c405cc47f6abea8d56cf1ba0b0 2024-11-16T10:24:05,967 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/724507c405cc47f6abea8d56cf1ba0b0, entries=150, sequenceid=322, filesize=12.0 K 2024-11-16T10:24:05,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/ef634949411e4f0792ed60d685716beb as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/ef634949411e4f0792ed60d685716beb 2024-11-16T10:24:05,975 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/ef634949411e4f0792ed60d685716beb, entries=150, sequenceid=322, filesize=12.0 K 2024-11-16T10:24:05,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/4b158d8efdba4f1b950e63b36c0a3e6f as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/4b158d8efdba4f1b950e63b36c0a3e6f 2024-11-16T10:24:05,983 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/4b158d8efdba4f1b950e63b36c0a3e6f, entries=150, sequenceid=322, filesize=12.0 K 2024-11-16T10:24:05,985 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for e63c645d595c80b0e763eedbebd195b2 in 554ms, sequenceid=322, compaction requested=true 2024-11-16T10:24:05,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:05,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:05,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-16T10:24:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-16T10:24:05,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-16T10:24:05,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2680 sec 2024-11-16T10:24:05,993 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.2750 sec 2024-11-16T10:24:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:24:06,030 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:24:06,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e63c645d595c80b0e763eedbebd195b2 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-16T10:24:06,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/30b1f880edee40c9bc8c80bcb3fcaf50 is 50, key is test_row_0/A:col10/1731752645705/Put/seqid=0 2024-11-16T10:24:06,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752706044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:06,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752706044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:06,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:06,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741906_1082 (size=17181) 2024-11-16T10:24:06,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:06,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:06,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:06,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752706156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:06,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:06,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:06,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:06,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752706156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:06,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:06,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:06,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:06,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752706367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:06,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:06,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:06,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:06,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752706367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:06,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:06,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:06,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:06,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/30b1f880edee40c9bc8c80bcb3fcaf50 2024-11-16T10:24:06,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/38ed5e63b78144959279121205348170 is 50, key is test_row_0/B:col10/1731752645705/Put/seqid=0 2024-11-16T10:24:06,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741907_1083 (size=12301) 2024-11-16T10:24:06,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:06,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752706676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752706676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:06,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:06,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:06,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/38ed5e63b78144959279121205348170 2024-11-16T10:24:06,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/9ac1febc1ab042f0af828f2f28d583e0 is 50, key is test_row_0/C:col10/1731752645705/Put/seqid=0 2024-11-16T10:24:06,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741908_1084 (size=12301) 2024-11-16T10:24:06,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/9ac1febc1ab042f0af828f2f28d583e0 2024-11-16T10:24:06,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/30b1f880edee40c9bc8c80bcb3fcaf50 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/30b1f880edee40c9bc8c80bcb3fcaf50 2024-11-16T10:24:06,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/30b1f880edee40c9bc8c80bcb3fcaf50, entries=250, sequenceid=352, filesize=16.8 K 2024-11-16T10:24:06,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/38ed5e63b78144959279121205348170 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/38ed5e63b78144959279121205348170 2024-11-16T10:24:06,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/38ed5e63b78144959279121205348170, entries=150, sequenceid=352, filesize=12.0 K 2024-11-16T10:24:06,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/9ac1febc1ab042f0af828f2f28d583e0 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/9ac1febc1ab042f0af828f2f28d583e0 2024-11-16T10:24:06,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/9ac1febc1ab042f0af828f2f28d583e0, entries=150, sequenceid=352, filesize=12.0 K 2024-11-16T10:24:06,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=26.84 KB/27480 for e63c645d595c80b0e763eedbebd195b2 in 954ms, sequenceid=352, compaction requested=true 2024-11-16T10:24:06,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:06,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e63c645d595c80b0e763eedbebd195b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:24:06,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:06,985 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:24:06,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e63c645d595c80b0e763eedbebd195b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:24:06,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:06,985 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:24:06,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e63c645d595c80b0e763eedbebd195b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:24:06,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:24:06,986 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:24:06,986 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): e63c645d595c80b0e763eedbebd195b2/B is initiating minor compaction (all files) 2024-11-16T10:24:06,986 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e63c645d595c80b0e763eedbebd195b2/B in TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:06,986 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/b8c4eff5c7344b5892e22817096a04c8, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/4567316d21df4e498e53add9e39861c9, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/ef634949411e4f0792ed60d685716beb, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/38ed5e63b78144959279121205348170] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp, totalSize=48.7 K 2024-11-16T10:24:06,988 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting b8c4eff5c7344b5892e22817096a04c8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731752643549 2024-11-16T10:24:06,988 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57206 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:24:06,988 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 4567316d21df4e498e53add9e39861c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731752644480 2024-11-16T10:24:06,988 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): e63c645d595c80b0e763eedbebd195b2/A is initiating minor compaction (all files) 2024-11-16T10:24:06,988 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e63c645d595c80b0e763eedbebd195b2/A in TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:06,988 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/a9387214362447ddbae78595a2d178cd, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/d17eab35c4f046508bece576327e7d5a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/724507c405cc47f6abea8d56cf1ba0b0, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/30b1f880edee40c9bc8c80bcb3fcaf50] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp, totalSize=55.9 K 2024-11-16T10:24:06,989 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting ef634949411e4f0792ed60d685716beb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731752644492 2024-11-16T10:24:06,989 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9387214362447ddbae78595a2d178cd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731752643549 2024-11-16T10:24:06,990 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 38ed5e63b78144959279121205348170, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731752645690 2024-11-16T10:24:06,991 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting d17eab35c4f046508bece576327e7d5a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731752643683 2024-11-16T10:24:06,992 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 724507c405cc47f6abea8d56cf1ba0b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731752644492 2024-11-16T10:24:06,992 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30b1f880edee40c9bc8c80bcb3fcaf50, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731752645690 2024-11-16T10:24:07,013 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e63c645d595c80b0e763eedbebd195b2#B#compaction#73 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:07,013 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e63c645d595c80b0e763eedbebd195b2#A#compaction#72 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:07,013 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/427daca9446d4107a26be36b6752283d is 50, key is test_row_0/B:col10/1731752645705/Put/seqid=0 2024-11-16T10:24:07,015 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/0213d015c1f34893a35751c24f33308d is 50, key is test_row_0/A:col10/1731752645705/Put/seqid=0 2024-11-16T10:24:07,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741910_1086 (size=13119) 2024-11-16T10:24:07,028 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/0213d015c1f34893a35751c24f33308d as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/0213d015c1f34893a35751c24f33308d 2024-11-16T10:24:07,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741909_1085 (size=13119) 2024-11-16T10:24:07,042 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e63c645d595c80b0e763eedbebd195b2/A of e63c645d595c80b0e763eedbebd195b2 into 0213d015c1f34893a35751c24f33308d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:07,042 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:07,043 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., storeName=e63c645d595c80b0e763eedbebd195b2/A, priority=12, startTime=1731752646985; duration=0sec 2024-11-16T10:24:07,043 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/427daca9446d4107a26be36b6752283d as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/427daca9446d4107a26be36b6752283d 2024-11-16T10:24:07,043 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:24:07,044 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e63c645d595c80b0e763eedbebd195b2:A 2024-11-16T10:24:07,044 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:24:07,047 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:24:07,048 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): e63c645d595c80b0e763eedbebd195b2/C is initiating minor compaction (all files) 2024-11-16T10:24:07,048 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e63c645d595c80b0e763eedbebd195b2/C in TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:07,048 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/e46269ee704444c8b4b96be08091e572, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/a25f9ddc0a4c49ea849d4bb444516c19, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/4b158d8efdba4f1b950e63b36c0a3e6f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/9ac1febc1ab042f0af828f2f28d583e0] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp, totalSize=48.7 K 2024-11-16T10:24:07,048 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting e46269ee704444c8b4b96be08091e572, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731752643549 2024-11-16T10:24:07,050 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting a25f9ddc0a4c49ea849d4bb444516c19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731752644480 2024-11-16T10:24:07,051 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b158d8efdba4f1b950e63b36c0a3e6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731752644492 2024-11-16T10:24:07,052 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ac1febc1ab042f0af828f2f28d583e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731752645690 2024-11-16T10:24:07,054 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e63c645d595c80b0e763eedbebd195b2/B of e63c645d595c80b0e763eedbebd195b2 into 427daca9446d4107a26be36b6752283d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:07,054 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:07,054 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., storeName=e63c645d595c80b0e763eedbebd195b2/B, priority=12, startTime=1731752646985; duration=0sec 2024-11-16T10:24:07,054 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:07,054 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e63c645d595c80b0e763eedbebd195b2:B 2024-11-16T10:24:07,066 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e63c645d595c80b0e763eedbebd195b2#C#compaction#74 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:07,067 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/cd87a031c11c4335a322583944139e6e is 50, key is test_row_0/C:col10/1731752645705/Put/seqid=0 2024-11-16T10:24:07,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741911_1087 (size=13119) 2024-11-16T10:24:07,081 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/cd87a031c11c4335a322583944139e6e as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/cd87a031c11c4335a322583944139e6e 2024-11-16T10:24:07,090 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e63c645d595c80b0e763eedbebd195b2/C of e63c645d595c80b0e763eedbebd195b2 into cd87a031c11c4335a322583944139e6e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:07,090 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:07,090 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., storeName=e63c645d595c80b0e763eedbebd195b2/C, priority=12, startTime=1731752646985; duration=0sec 2024-11-16T10:24:07,091 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:07,091 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e63c645d595c80b0e763eedbebd195b2:C 2024-11-16T10:24:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:24:07,224 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:24:07,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e63c645d595c80b0e763eedbebd195b2 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T10:24:07,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/5ceeab731970406aae29b33e29e5ee3a is 50, key is test_row_0/A:col10/1731752647190/Put/seqid=0 2024-11-16T10:24:07,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741912_1088 (size=17181) 2024-11-16T10:24:07,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:07,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752707285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:07,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752707285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:07,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:07,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:07,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:07,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:07,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752707396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:07,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:07,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:07,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:07,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:07,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752707396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:07,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:07,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:07,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:07,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:07,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752707606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:07,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:07,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752707606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:07,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:07,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:07,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:07,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:07,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:07,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:07,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/5ceeab731970406aae29b33e29e5ee3a 2024-11-16T10:24:07,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/554a0ac19c9c4a3a8fad49a259a36a3f is 50, key is test_row_0/B:col10/1731752647190/Put/seqid=0 2024-11-16T10:24:07,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741913_1089 (size=12301) 2024-11-16T10:24:07,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:07,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752707866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:07,867 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_2' in e63c645d595c80b0e763eedbebd195b2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:24:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:07,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T10:24:07,876 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:24:07,877 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:24:07,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-16T10:24:07,879 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:24:07,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T10:24:07,880 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:24:07,880 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:24:07,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:07,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752707917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:07,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:07,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:07,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:07,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:07,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752707923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:07,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:07,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:07,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:07,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T10:24:08,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T10:24:08,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:08,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:08,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:08,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:08,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:08,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:08,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/554a0ac19c9c4a3a8fad49a259a36a3f 2024-11-16T10:24:08,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/72abd22a403c45ffa5bd2a9861bb8e05 is 50, key is test_row_0/C:col10/1731752647190/Put/seqid=0 2024-11-16T10:24:08,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741914_1090 (size=12301) 2024-11-16T10:24:08,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T10:24:08,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:08,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:08,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:08,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:08,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:08,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:08,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T10:24:08,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T10:24:08,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:08,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:08,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:08,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:08,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:08,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:08,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752708426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:08,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:08,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:08,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:08,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:08,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752708436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:08,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:08,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:08,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:08,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T10:24:08,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:08,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:08,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:08,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:08,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:08,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:08,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/72abd22a403c45ffa5bd2a9861bb8e05 2024-11-16T10:24:08,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T10:24:08,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/5ceeab731970406aae29b33e29e5ee3a as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/5ceeab731970406aae29b33e29e5ee3a 2024-11-16T10:24:08,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/5ceeab731970406aae29b33e29e5ee3a, entries=250, sequenceid=367, filesize=16.8 K 2024-11-16T10:24:08,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/554a0ac19c9c4a3a8fad49a259a36a3f as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/554a0ac19c9c4a3a8fad49a259a36a3f 2024-11-16T10:24:08,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/554a0ac19c9c4a3a8fad49a259a36a3f, entries=150, sequenceid=367, filesize=12.0 K 2024-11-16T10:24:08,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/72abd22a403c45ffa5bd2a9861bb8e05 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/72abd22a403c45ffa5bd2a9861bb8e05 2024-11-16T10:24:08,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/72abd22a403c45ffa5bd2a9861bb8e05, entries=150, sequenceid=367, filesize=12.0 K 2024-11-16T10:24:08,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e63c645d595c80b0e763eedbebd195b2 in 1305ms, sequenceid=367, compaction requested=false 2024-11-16T10:24:08,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:08,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T10:24:08,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:08,653 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing e63c645d595c80b0e763eedbebd195b2 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T10:24:08,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/9b8ae0408b1742f5be2789919025d7db is 50, key is test_row_0/A:col10/1731752647282/Put/seqid=0 2024-11-16T10:24:08,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741915_1091 (size=12301) 2024-11-16T10:24:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T10:24:09,068 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/9b8ae0408b1742f5be2789919025d7db 2024-11-16T10:24:09,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/dd8c02fd184f45bd92b7a7cee80d1103 is 50, key is test_row_0/B:col10/1731752647282/Put/seqid=0 2024-11-16T10:24:09,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741916_1092 (size=12301) 2024-11-16T10:24:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:24:09,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:09,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752709467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:09,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:09,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:09,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:09,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752709469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:09,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:09,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:09,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:09,490 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/dd8c02fd184f45bd92b7a7cee80d1103 2024-11-16T10:24:09,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/5d068b499e7b4d2d9cd79693b6889950 is 50, key is test_row_0/C:col10/1731752647282/Put/seqid=0 2024-11-16T10:24:09,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741917_1093 (size=12301) 2024-11-16T10:24:09,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:09,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752709576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:09,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:09,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:09,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:09,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:09,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752709576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:09,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:09,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:09,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:09,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:09,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752709786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:09,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:09,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752709786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:09,921 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/5d068b499e7b4d2d9cd79693b6889950 2024-11-16T10:24:09,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/9b8ae0408b1742f5be2789919025d7db as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/9b8ae0408b1742f5be2789919025d7db 2024-11-16T10:24:09,939 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/9b8ae0408b1742f5be2789919025d7db, entries=150, sequenceid=391, filesize=12.0 K 2024-11-16T10:24:09,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/dd8c02fd184f45bd92b7a7cee80d1103 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/dd8c02fd184f45bd92b7a7cee80d1103 2024-11-16T10:24:09,950 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/dd8c02fd184f45bd92b7a7cee80d1103, entries=150, sequenceid=391, filesize=12.0 K 2024-11-16T10:24:09,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/5d068b499e7b4d2d9cd79693b6889950 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/5d068b499e7b4d2d9cd79693b6889950 2024-11-16T10:24:09,960 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/5d068b499e7b4d2d9cd79693b6889950, entries=150, sequenceid=391, filesize=12.0 K 2024-11-16T10:24:09,961 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for e63c645d595c80b0e763eedbebd195b2 in 1308ms, sequenceid=391, compaction requested=true 2024-11-16T10:24:09,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:09,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:09,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-16T10:24:09,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-16T10:24:09,964 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-16T10:24:09,964 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0830 sec 2024-11-16T10:24:09,968 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.0880 sec 2024-11-16T10:24:10,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T10:24:10,037 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:24:10,037 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:24:10,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-16T10:24:10,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T10:24:10,039 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:24:10,041 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:24:10,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:24:10,099 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:24:10,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e63c645d595c80b0e763eedbebd195b2 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T10:24:10,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:24:10,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/a950908d6f8747ec9fbe9d04d975fbc4 is 50, key is test_row_0/A:col10/1731752649465/Put/seqid=0 2024-11-16T10:24:10,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741918_1094 (size=14741) 2024-11-16T10:24:10,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/a950908d6f8747ec9fbe9d04d975fbc4 2024-11-16T10:24:10,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/33c50bf8a52c48d4844bd4aee91e0ed2 is 50, key is test_row_0/B:col10/1731752649465/Put/seqid=0 2024-11-16T10:24:10,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741919_1095 (size=12301) 2024-11-16T10:24:10,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/33c50bf8a52c48d4844bd4aee91e0ed2 2024-11-16T10:24:10,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T10:24:10,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/b0027f143972466e95b15d742ebe5ada is 50, key is test_row_0/C:col10/1731752649465/Put/seqid=0 2024-11-16T10:24:10,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741920_1096 (size=12301) 2024-11-16T10:24:10,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-16T10:24:10,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:10,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:10,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:10,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:10,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:10,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:10,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:10,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752710193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:10,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:10,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752710197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:10,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:10,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:10,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:10,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:10,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752710306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:10,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:10,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:10,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:10,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:10,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752710306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:10,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:10,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:10,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:10,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-16T10:24:10,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:10,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:10,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:10,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:10,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:10,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T10:24:10,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-16T10:24:10,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:10,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:10,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:10,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:10,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:10,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:10,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:10,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752710516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:10,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752710516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:10,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:10,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:10,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:10,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:10,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:10,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:10,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/b0027f143972466e95b15d742ebe5ada 2024-11-16T10:24:10,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/a950908d6f8747ec9fbe9d04d975fbc4 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/a950908d6f8747ec9fbe9d04d975fbc4 2024-11-16T10:24:10,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/a950908d6f8747ec9fbe9d04d975fbc4, entries=200, sequenceid=404, filesize=14.4 K 2024-11-16T10:24:10,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/33c50bf8a52c48d4844bd4aee91e0ed2 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/33c50bf8a52c48d4844bd4aee91e0ed2 2024-11-16T10:24:10,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/33c50bf8a52c48d4844bd4aee91e0ed2, entries=150, sequenceid=404, filesize=12.0 K 2024-11-16T10:24:10,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/b0027f143972466e95b15d742ebe5ada as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/b0027f143972466e95b15d742ebe5ada 2024-11-16T10:24:10,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/b0027f143972466e95b15d742ebe5ada, entries=150, sequenceid=404, filesize=12.0 K 2024-11-16T10:24:10,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for e63c645d595c80b0e763eedbebd195b2 in 512ms, sequenceid=404, compaction requested=true 2024-11-16T10:24:10,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:10,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e63c645d595c80b0e763eedbebd195b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:24:10,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:10,611 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:24:10,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e63c645d595c80b0e763eedbebd195b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:24:10,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:24:10,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e63c645d595c80b0e763eedbebd195b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:24:10,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T10:24:10,612 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:24:10,615 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:24:10,615 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57342 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:24:10,615 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): e63c645d595c80b0e763eedbebd195b2/B is initiating minor compaction (all files) 2024-11-16T10:24:10,615 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): e63c645d595c80b0e763eedbebd195b2/A is initiating minor compaction (all files) 2024-11-16T10:24:10,615 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e63c645d595c80b0e763eedbebd195b2/B in TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:10,615 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e63c645d595c80b0e763eedbebd195b2/A in TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:10,615 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/0213d015c1f34893a35751c24f33308d, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/5ceeab731970406aae29b33e29e5ee3a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/9b8ae0408b1742f5be2789919025d7db, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/a950908d6f8747ec9fbe9d04d975fbc4] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp, totalSize=56.0 K 2024-11-16T10:24:10,615 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/427daca9446d4107a26be36b6752283d, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/554a0ac19c9c4a3a8fad49a259a36a3f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/dd8c02fd184f45bd92b7a7cee80d1103, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/33c50bf8a52c48d4844bd4aee91e0ed2] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp, totalSize=48.8 K 2024-11-16T10:24:10,616 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 427daca9446d4107a26be36b6752283d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731752645690 2024-11-16T10:24:10,617 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0213d015c1f34893a35751c24f33308d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731752645690 2024-11-16T10:24:10,617 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 554a0ac19c9c4a3a8fad49a259a36a3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731752647188 2024-11-16T10:24:10,617 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ceeab731970406aae29b33e29e5ee3a, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731752647188 2024-11-16T10:24:10,618 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting dd8c02fd184f45bd92b7a7cee80d1103, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731752647276 2024-11-16T10:24:10,619 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b8ae0408b1742f5be2789919025d7db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731752647276 2024-11-16T10:24:10,619 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 33c50bf8a52c48d4844bd4aee91e0ed2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731752649452 2024-11-16T10:24:10,619 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting a950908d6f8747ec9fbe9d04d975fbc4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731752649452 2024-11-16T10:24:10,633 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e63c645d595c80b0e763eedbebd195b2#A#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:10,634 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/3611204636e34c1089f3178bccbf01fe is 50, key is test_row_0/A:col10/1731752649465/Put/seqid=0 2024-11-16T10:24:10,634 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e63c645d595c80b0e763eedbebd195b2#B#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:10,635 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/5c08bac83efa4e59bf25b53bfddc4dd0 is 50, key is test_row_0/B:col10/1731752649465/Put/seqid=0 2024-11-16T10:24:10,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741921_1097 (size=13255) 2024-11-16T10:24:10,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741922_1098 (size=13255) 2024-11-16T10:24:10,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-16T10:24:10,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:10,654 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing e63c645d595c80b0e763eedbebd195b2 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T10:24:10,661 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/5c08bac83efa4e59bf25b53bfddc4dd0 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/5c08bac83efa4e59bf25b53bfddc4dd0 2024-11-16T10:24:10,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/a5c2dd24d3ff4688a9e2e60053d61049 is 50, key is test_row_0/A:col10/1731752650149/Put/seqid=0 2024-11-16T10:24:10,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T10:24:10,670 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e63c645d595c80b0e763eedbebd195b2/B of e63c645d595c80b0e763eedbebd195b2 into 5c08bac83efa4e59bf25b53bfddc4dd0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:10,670 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:10,670 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., storeName=e63c645d595c80b0e763eedbebd195b2/B, priority=12, startTime=1731752650611; duration=0sec 2024-11-16T10:24:10,670 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:24:10,670 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e63c645d595c80b0e763eedbebd195b2:B 2024-11-16T10:24:10,671 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:24:10,673 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:24:10,673 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): e63c645d595c80b0e763eedbebd195b2/C is initiating minor compaction (all files) 2024-11-16T10:24:10,673 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e63c645d595c80b0e763eedbebd195b2/C in TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:10,673 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/cd87a031c11c4335a322583944139e6e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/72abd22a403c45ffa5bd2a9861bb8e05, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/5d068b499e7b4d2d9cd79693b6889950, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/b0027f143972466e95b15d742ebe5ada] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp, totalSize=48.8 K 2024-11-16T10:24:10,674 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting cd87a031c11c4335a322583944139e6e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731752645690 2024-11-16T10:24:10,675 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 72abd22a403c45ffa5bd2a9861bb8e05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731752647188 2024-11-16T10:24:10,675 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d068b499e7b4d2d9cd79693b6889950, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731752647276 2024-11-16T10:24:10,676 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting b0027f143972466e95b15d742ebe5ada, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731752649452 2024-11-16T10:24:10,699 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e63c645d595c80b0e763eedbebd195b2#C#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:10,700 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/2a608a767a574401b3ebf531e7554ded is 50, key is test_row_0/C:col10/1731752649465/Put/seqid=0 2024-11-16T10:24:10,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741923_1099 (size=12301) 2024-11-16T10:24:10,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741924_1100 (size=13255) 2024-11-16T10:24:10,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:24:10,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:10,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:10,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752710843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:10,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:10,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:10,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:10,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752710844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:10,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:10,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:10,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:10,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:10,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:10,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752710956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:10,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:10,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752710956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:10,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:10,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:10,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:11,057 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/3611204636e34c1089f3178bccbf01fe as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/3611204636e34c1089f3178bccbf01fe 2024-11-16T10:24:11,065 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e63c645d595c80b0e763eedbebd195b2/A of e63c645d595c80b0e763eedbebd195b2 into 3611204636e34c1089f3178bccbf01fe(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:11,065 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:11,065 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., storeName=e63c645d595c80b0e763eedbebd195b2/A, priority=12, startTime=1731752650611; duration=0sec 2024-11-16T10:24:11,065 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:11,065 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e63c645d595c80b0e763eedbebd195b2:A 2024-11-16T10:24:11,106 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/a5c2dd24d3ff4688a9e2e60053d61049 2024-11-16T10:24:11,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/6f4554f932c14d2493ab1c6979ab2690 is 50, key is test_row_0/B:col10/1731752650149/Put/seqid=0 2024-11-16T10:24:11,126 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/2a608a767a574401b3ebf531e7554ded as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/2a608a767a574401b3ebf531e7554ded 2024-11-16T10:24:11,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741925_1101 (size=12301) 2024-11-16T10:24:11,132 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/6f4554f932c14d2493ab1c6979ab2690 2024-11-16T10:24:11,133 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e63c645d595c80b0e763eedbebd195b2/C of e63c645d595c80b0e763eedbebd195b2 into 2a608a767a574401b3ebf531e7554ded(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:11,133 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:11,133 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., storeName=e63c645d595c80b0e763eedbebd195b2/C, priority=12, startTime=1731752650611; duration=0sec 2024-11-16T10:24:11,133 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:11,133 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e63c645d595c80b0e763eedbebd195b2:C 2024-11-16T10:24:11,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/b2025b4700b646ef9b38512471348e43 is 50, key is test_row_0/C:col10/1731752650149/Put/seqid=0 2024-11-16T10:24:11,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741926_1102 (size=12301) 2024-11-16T10:24:11,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:11,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:11,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752711166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:11,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752711166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:11,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:11,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:11,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:11,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:11,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:11,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:11,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T10:24:11,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:11,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752711476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:11,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:11,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:11,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:11,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:11,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752711476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:11,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:11,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:11,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:11,552 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/b2025b4700b646ef9b38512471348e43 2024-11-16T10:24:11,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/a5c2dd24d3ff4688a9e2e60053d61049 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/a5c2dd24d3ff4688a9e2e60053d61049 2024-11-16T10:24:11,616 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/a5c2dd24d3ff4688a9e2e60053d61049, entries=150, sequenceid=427, filesize=12.0 K 2024-11-16T10:24:11,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/6f4554f932c14d2493ab1c6979ab2690 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/6f4554f932c14d2493ab1c6979ab2690 2024-11-16T10:24:11,624 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/6f4554f932c14d2493ab1c6979ab2690, entries=150, sequenceid=427, filesize=12.0 K 2024-11-16T10:24:11,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/b2025b4700b646ef9b38512471348e43 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/b2025b4700b646ef9b38512471348e43 2024-11-16T10:24:11,630 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/b2025b4700b646ef9b38512471348e43, entries=150, sequenceid=427, filesize=12.0 K 2024-11-16T10:24:11,631 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for e63c645d595c80b0e763eedbebd195b2 in 977ms, sequenceid=427, compaction requested=false 2024-11-16T10:24:11,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:11,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:11,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-16T10:24:11,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-16T10:24:11,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-16T10:24:11,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5920 sec 2024-11-16T10:24:11,637 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.5990 sec 2024-11-16T10:24:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:24:11,909 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:24:11,910 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e63c645d595c80b0e763eedbebd195b2 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-16T10:24:11,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/1dfc9db6899d42d9920698f9f4cc7918 is 50, key is test_row_0/A:col10/1731752650839/Put/seqid=0 2024-11-16T10:24:11,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741927_1103 (size=14741) 2024-11-16T10:24:11,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752711981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:11,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:11,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752711986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:11,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:11,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:11,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:11,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752711986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:11,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:11,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:11,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:11,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:12,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752712086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T10:24:12,186 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:24:12,186 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:24:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-16T10:24:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T10:24:12,188 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:24:12,189 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:24:12,189 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:24:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T10:24:12,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752712296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:12,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:12,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:12,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:12,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/1dfc9db6899d42d9920698f9f4cc7918 2024-11-16T10:24:12,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/75f747d6cdc247e4aee572831fa0597f is 50, key is test_row_0/B:col10/1731752650839/Put/seqid=0 2024-11-16T10:24:12,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741928_1104 (size=12301) 2024-11-16T10:24:12,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T10:24:12,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:12,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:12,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:12,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:12,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:12,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T10:24:12,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:12,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:12,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:12,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:12,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T10:24:12,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:12,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752712606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:12,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:12,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:12,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:12,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T10:24:12,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:12,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:12,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:12,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:12,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:12,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/75f747d6cdc247e4aee572831fa0597f 2024-11-16T10:24:12,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/094bc01b5f65416cba32d3e7c91938df is 50, key is test_row_0/C:col10/1731752650839/Put/seqid=0 2024-11-16T10:24:12,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741929_1105 (size=12301) 2024-11-16T10:24:12,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T10:24:12,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:12,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:12,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:12,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:12,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:12,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:12,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T10:24:12,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T10:24:12,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:12,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:12,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:12,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:12,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:12,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:12,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:12,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37590 deadline: 1731752712996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:12,998 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_1' in e63c645d595c80b0e763eedbebd195b2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:24:12,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:12,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:12,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:13,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:13,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37638 deadline: 1731752713006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:13,008 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_1' in e63c645d595c80b0e763eedbebd195b2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:24:13,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:13,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T10:24:13,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:13,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. as already flushing 2024-11-16T10:24:13,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:13,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:13,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:13,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:13,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:13,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37606 deadline: 1731752713116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:13,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:13,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:13,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:13,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/094bc01b5f65416cba32d3e7c91938df 2024-11-16T10:24:13,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/1dfc9db6899d42d9920698f9f4cc7918 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/1dfc9db6899d42d9920698f9f4cc7918 2024-11-16T10:24:13,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/1dfc9db6899d42d9920698f9f4cc7918, entries=200, sequenceid=444, filesize=14.4 K 2024-11-16T10:24:13,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/75f747d6cdc247e4aee572831fa0597f as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/75f747d6cdc247e4aee572831fa0597f 2024-11-16T10:24:13,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/75f747d6cdc247e4aee572831fa0597f, entries=150, sequenceid=444, filesize=12.0 K 2024-11-16T10:24:13,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/094bc01b5f65416cba32d3e7c91938df as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/094bc01b5f65416cba32d3e7c91938df 2024-11-16T10:24:13,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/094bc01b5f65416cba32d3e7c91938df, entries=150, sequenceid=444, filesize=12.0 K 2024-11-16T10:24:13,171 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for e63c645d595c80b0e763eedbebd195b2 in 1261ms, sequenceid=444, compaction requested=true 2024-11-16T10:24:13,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:13,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e63c645d595c80b0e763eedbebd195b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:24:13,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:13,171 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:24:13,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e63c645d595c80b0e763eedbebd195b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:24:13,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:13,171 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:24:13,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e63c645d595c80b0e763eedbebd195b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:24:13,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:24:13,172 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:24:13,172 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:24:13,172 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): e63c645d595c80b0e763eedbebd195b2/B is initiating minor compaction (all files) 2024-11-16T10:24:13,172 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): e63c645d595c80b0e763eedbebd195b2/A is initiating minor compaction (all files) 2024-11-16T10:24:13,172 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e63c645d595c80b0e763eedbebd195b2/B in TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:13,172 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e63c645d595c80b0e763eedbebd195b2/A in TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:13,173 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/3611204636e34c1089f3178bccbf01fe, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/a5c2dd24d3ff4688a9e2e60053d61049, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/1dfc9db6899d42d9920698f9f4cc7918] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp, totalSize=39.4 K 2024-11-16T10:24:13,173 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/5c08bac83efa4e59bf25b53bfddc4dd0, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/6f4554f932c14d2493ab1c6979ab2690, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/75f747d6cdc247e4aee572831fa0597f] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp, totalSize=37.0 K 2024-11-16T10:24:13,173 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3611204636e34c1089f3178bccbf01fe, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731752649452 2024-11-16T10:24:13,173 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c08bac83efa4e59bf25b53bfddc4dd0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731752649452 2024-11-16T10:24:13,173 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5c2dd24d3ff4688a9e2e60053d61049, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731752650149 2024-11-16T10:24:13,173 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f4554f932c14d2493ab1c6979ab2690, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731752650149 2024-11-16T10:24:13,174 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dfc9db6899d42d9920698f9f4cc7918, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731752650829 2024-11-16T10:24:13,174 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 75f747d6cdc247e4aee572831fa0597f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731752650829 2024-11-16T10:24:13,181 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e63c645d595c80b0e763eedbebd195b2#B#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:13,181 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/7cdf63fb1aeb4253bb9ec64401eed7f2 is 50, key is test_row_0/B:col10/1731752650839/Put/seqid=0 2024-11-16T10:24:13,181 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e63c645d595c80b0e763eedbebd195b2#A#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:13,182 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/6b7562162f494bc7a8508612d51d6dfa is 50, key is test_row_0/A:col10/1731752650839/Put/seqid=0 2024-11-16T10:24:13,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741930_1106 (size=13357) 2024-11-16T10:24:13,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741931_1107 (size=13357) 2024-11-16T10:24:13,196 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/6b7562162f494bc7a8508612d51d6dfa as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/6b7562162f494bc7a8508612d51d6dfa 2024-11-16T10:24:13,201 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e63c645d595c80b0e763eedbebd195b2/A of e63c645d595c80b0e763eedbebd195b2 into 6b7562162f494bc7a8508612d51d6dfa(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:13,201 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:13,202 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., storeName=e63c645d595c80b0e763eedbebd195b2/A, priority=13, startTime=1731752653171; duration=0sec 2024-11-16T10:24:13,202 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:24:13,202 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e63c645d595c80b0e763eedbebd195b2:A 2024-11-16T10:24:13,202 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:24:13,203 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:24:13,203 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): e63c645d595c80b0e763eedbebd195b2/C is initiating minor compaction (all files) 2024-11-16T10:24:13,203 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e63c645d595c80b0e763eedbebd195b2/C in TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:13,203 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/2a608a767a574401b3ebf531e7554ded, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/b2025b4700b646ef9b38512471348e43, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/094bc01b5f65416cba32d3e7c91938df] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp, totalSize=37.0 K 2024-11-16T10:24:13,204 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a608a767a574401b3ebf531e7554ded, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731752649452 2024-11-16T10:24:13,204 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2025b4700b646ef9b38512471348e43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731752650149 2024-11-16T10:24:13,204 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 094bc01b5f65416cba32d3e7c91938df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731752650829 2024-11-16T10:24:13,212 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e63c645d595c80b0e763eedbebd195b2#C#compaction#95 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:13,212 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/ac7bbc1a01e040bcb18442a67e4d756b is 50, key is test_row_0/C:col10/1731752650839/Put/seqid=0 2024-11-16T10:24:13,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741932_1108 (size=13357) 2024-11-16T10:24:13,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T10:24:13,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:13,259 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing e63c645d595c80b0e763eedbebd195b2 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-16T10:24:13,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/9c902d53fbbc4bbcb2c2862724273696 is 50, key is test_row_0/A:col10/1731752651973/Put/seqid=0 2024-11-16T10:24:13,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741933_1109 (size=12301) 2024-11-16T10:24:13,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T10:24:13,592 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/7cdf63fb1aeb4253bb9ec64401eed7f2 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/7cdf63fb1aeb4253bb9ec64401eed7f2 2024-11-16T10:24:13,598 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e63c645d595c80b0e763eedbebd195b2/B of e63c645d595c80b0e763eedbebd195b2 into 7cdf63fb1aeb4253bb9ec64401eed7f2(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:13,598 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:13,598 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., storeName=e63c645d595c80b0e763eedbebd195b2/B, priority=13, startTime=1731752653171; duration=0sec 2024-11-16T10:24:13,598 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:13,598 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e63c645d595c80b0e763eedbebd195b2:B 2024-11-16T10:24:13,622 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/ac7bbc1a01e040bcb18442a67e4d756b as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/ac7bbc1a01e040bcb18442a67e4d756b 2024-11-16T10:24:13,628 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e63c645d595c80b0e763eedbebd195b2/C of e63c645d595c80b0e763eedbebd195b2 into ac7bbc1a01e040bcb18442a67e4d756b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:13,628 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:13,628 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., storeName=e63c645d595c80b0e763eedbebd195b2/C, priority=13, startTime=1731752653171; duration=0sec 2024-11-16T10:24:13,628 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:13,628 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e63c645d595c80b0e763eedbebd195b2:C 2024-11-16T10:24:13,668 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/9c902d53fbbc4bbcb2c2862724273696 2024-11-16T10:24:13,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/e2d2e440dbd447d78ababf57f281f279 is 50, key is test_row_0/B:col10/1731752651973/Put/seqid=0 2024-11-16T10:24:13,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741934_1110 (size=12301) 2024-11-16T10:24:13,686 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/e2d2e440dbd447d78ababf57f281f279 2024-11-16T10:24:13,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/6f628be58d0f4b5cb896e6e2223d08ad is 50, key is test_row_0/C:col10/1731752651973/Put/seqid=0 2024-11-16T10:24:13,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741935_1111 (size=12301) 2024-11-16T10:24:13,702 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/6f628be58d0f4b5cb896e6e2223d08ad 2024-11-16T10:24:13,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/9c902d53fbbc4bbcb2c2862724273696 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/9c902d53fbbc4bbcb2c2862724273696 2024-11-16T10:24:13,722 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/9c902d53fbbc4bbcb2c2862724273696, entries=150, sequenceid=467, filesize=12.0 K 2024-11-16T10:24:13,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/e2d2e440dbd447d78ababf57f281f279 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/e2d2e440dbd447d78ababf57f281f279 2024-11-16T10:24:13,729 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/e2d2e440dbd447d78ababf57f281f279, entries=150, sequenceid=467, filesize=12.0 K 2024-11-16T10:24:13,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/6f628be58d0f4b5cb896e6e2223d08ad as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/6f628be58d0f4b5cb896e6e2223d08ad 2024-11-16T10:24:13,736 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/6f628be58d0f4b5cb896e6e2223d08ad, entries=150, sequenceid=467, filesize=12.0 K 2024-11-16T10:24:13,738 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for e63c645d595c80b0e763eedbebd195b2 in 479ms, sequenceid=467, compaction requested=false 2024-11-16T10:24:13,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for e63c645d595c80b0e763eedbebd195b2: 2024-11-16T10:24:13,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:13,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-16T10:24:13,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-16T10:24:13,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-16T10:24:13,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5500 sec 2024-11-16T10:24:13,744 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.5560 sec 2024-11-16T10:24:13,940 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-16T10:24:13,940 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-16T10:24:13,940 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:24:13,940 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:24:13,940 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-16T10:24:13,940 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:24:13,941 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-16T10:24:13,941 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:24:13,944 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:13,944 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:13,944 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:13,944 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:13,944 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:13,944 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:13,945 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:13,945 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:24:13,945 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:13,945 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:24:13,945 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:24:13,945 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:24:14,131 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-16T10:24:14,131 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:24:14,131 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:14,132 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:14,132 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:24:14,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T10:24:14,338 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:24:14,462 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-16T10:24:14,462 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:24:14,462 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:14,462 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:14,463 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:24:14,551 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-16T10:24:14,551 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:24:14,551 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:14,552 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:14,552 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:24:15,022 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-16T10:24:15,022 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-16T10:24:15,022 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:24:15,022 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:24:15,022 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:15,023 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:15,023 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:15,023 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:15,023 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:24:15,023 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:24:15,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T10:24:15,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-16T10:24:15,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-16T10:24:15,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-16T10:24:15,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 211 2024-11-16T10:24:15,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-16T10:24:15,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T10:24:15,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3585 2024-11-16T10:24:15,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3612 2024-11-16T10:24:15,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T10:24:15,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1647 2024-11-16T10:24:15,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4937 rows 2024-11-16T10:24:15,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1650 2024-11-16T10:24:15,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4946 rows 2024-11-16T10:24:15,025 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T10:24:15,025 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T10:24:15,025 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:15,025 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:15,025 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:24:15,031 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T10:24:15,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T10:24:15,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T10:24:15,038 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752655038"}]},"ts":"1731752655038"} 2024-11-16T10:24:15,040 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T10:24:15,040 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T10:24:15,042 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T10:24:15,047 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e63c645d595c80b0e763eedbebd195b2, UNASSIGN}] 2024-11-16T10:24:15,048 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e63c645d595c80b0e763eedbebd195b2, UNASSIGN 2024-11-16T10:24:15,050 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=e63c645d595c80b0e763eedbebd195b2, regionState=CLOSING, regionLocation=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:15,053 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e63c645d595c80b0e763eedbebd195b2, UNASSIGN because future has completed 2024-11-16T10:24:15,053 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T10:24:15,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506}] 2024-11-16T10:24:15,095 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T10:24:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T10:24:15,213 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:24:15,213 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T10:24:15,214 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing e63c645d595c80b0e763eedbebd195b2, disabling compactions & flushes 2024-11-16T10:24:15,214 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:15,215 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:15,215 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. after waiting 0 ms 2024-11-16T10:24:15,215 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:15,215 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing e63c645d595c80b0e763eedbebd195b2 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-16T10:24:15,222 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/b5b460020fe54bb5a7fdda74bc3529fa is 50, key is test_row_0/A:col10/1731752654458/Put/seqid=0 2024-11-16T10:24:15,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741936_1112 (size=12301) 2024-11-16T10:24:15,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T10:24:15,628 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/b5b460020fe54bb5a7fdda74bc3529fa 2024-11-16T10:24:15,639 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/f5237245ec864f04b5f6fea811e3e6f5 is 50, key is test_row_0/B:col10/1731752654458/Put/seqid=0 2024-11-16T10:24:15,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741937_1113 (size=12301) 2024-11-16T10:24:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T10:24:16,052 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/f5237245ec864f04b5f6fea811e3e6f5 2024-11-16T10:24:16,069 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/0930a8ee7b904f33b587046c4caf3bc4 is 50, key is test_row_0/C:col10/1731752654458/Put/seqid=0 2024-11-16T10:24:16,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741938_1114 (size=12301) 2024-11-16T10:24:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T10:24:16,474 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/0930a8ee7b904f33b587046c4caf3bc4 2024-11-16T10:24:16,487 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/A/b5b460020fe54bb5a7fdda74bc3529fa as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/b5b460020fe54bb5a7fdda74bc3529fa 2024-11-16T10:24:16,493 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/b5b460020fe54bb5a7fdda74bc3529fa, entries=150, sequenceid=477, filesize=12.0 K 2024-11-16T10:24:16,494 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/B/f5237245ec864f04b5f6fea811e3e6f5 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/f5237245ec864f04b5f6fea811e3e6f5 2024-11-16T10:24:16,500 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/f5237245ec864f04b5f6fea811e3e6f5, entries=150, sequenceid=477, filesize=12.0 K 2024-11-16T10:24:16,501 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/.tmp/C/0930a8ee7b904f33b587046c4caf3bc4 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/0930a8ee7b904f33b587046c4caf3bc4 2024-11-16T10:24:16,506 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/0930a8ee7b904f33b587046c4caf3bc4, entries=150, sequenceid=477, filesize=12.0 K 2024-11-16T10:24:16,507 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for e63c645d595c80b0e763eedbebd195b2 in 1292ms, sequenceid=477, compaction requested=true 2024-11-16T10:24:16,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/0b5b99bc0f1846289d0c50f9fda71e76, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/440807c8317f49a582f384d8bc42dc09, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/93debc24c7db46ec87d0bb05f3d1bb77, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/f10ee9814a4643cdb0c6c8b563fd214e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/32cc2adb0fed4323afa72e7f59543660, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/1365ef95725f4f6c920adbe1d813461a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/1419b79484d44bfb94307530ab54b9eb, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/088658cd19344f39b9bd111e05e964ba, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/d39f96f86e4d4cb0b780c5c00e91ac03, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/3dd8d545cc664fac893058bf4d884e65, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/3b45821b5f7c435ebb897621fe1b2c36, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/d6a6e5b4d1c149e980a37062f3001187, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/606ceb7832094eb68a9d828122834b43, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/533350ee8d8b451e8e9a38ab9873979b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/f8f13c7b51e34fc7ae49d5309355a28b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/e7b52a00abfc423dbce919b951aa299d, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/cc349a464bdf423284cd4fda424feb0e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/3a7c5c1ea7784149b72131dd01ef3b38, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/007c6c5f05184c2f82f7b4ad611f67f0, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/0fb5d05194a54cfc99c1144c30221d1b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/a9387214362447ddbae78595a2d178cd, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/d17eab35c4f046508bece576327e7d5a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/724507c405cc47f6abea8d56cf1ba0b0, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/30b1f880edee40c9bc8c80bcb3fcaf50, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/0213d015c1f34893a35751c24f33308d, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/5ceeab731970406aae29b33e29e5ee3a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/9b8ae0408b1742f5be2789919025d7db, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/a950908d6f8747ec9fbe9d04d975fbc4, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/3611204636e34c1089f3178bccbf01fe, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/a5c2dd24d3ff4688a9e2e60053d61049, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/1dfc9db6899d42d9920698f9f4cc7918] to archive 2024-11-16T10:24:16,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:24:16,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/0b5b99bc0f1846289d0c50f9fda71e76 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/0b5b99bc0f1846289d0c50f9fda71e76 2024-11-16T10:24:16,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/440807c8317f49a582f384d8bc42dc09 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/440807c8317f49a582f384d8bc42dc09 2024-11-16T10:24:16,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/93debc24c7db46ec87d0bb05f3d1bb77 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/93debc24c7db46ec87d0bb05f3d1bb77 2024-11-16T10:24:16,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/f10ee9814a4643cdb0c6c8b563fd214e to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/f10ee9814a4643cdb0c6c8b563fd214e 2024-11-16T10:24:16,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/32cc2adb0fed4323afa72e7f59543660 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/32cc2adb0fed4323afa72e7f59543660 2024-11-16T10:24:16,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/1365ef95725f4f6c920adbe1d813461a to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/1365ef95725f4f6c920adbe1d813461a 2024-11-16T10:24:16,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/1419b79484d44bfb94307530ab54b9eb to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/1419b79484d44bfb94307530ab54b9eb 2024-11-16T10:24:16,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/088658cd19344f39b9bd111e05e964ba to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/088658cd19344f39b9bd111e05e964ba 2024-11-16T10:24:16,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/d39f96f86e4d4cb0b780c5c00e91ac03 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/d39f96f86e4d4cb0b780c5c00e91ac03 2024-11-16T10:24:16,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/3dd8d545cc664fac893058bf4d884e65 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/3dd8d545cc664fac893058bf4d884e65 2024-11-16T10:24:16,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/3b45821b5f7c435ebb897621fe1b2c36 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/3b45821b5f7c435ebb897621fe1b2c36 2024-11-16T10:24:16,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/d6a6e5b4d1c149e980a37062f3001187 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/d6a6e5b4d1c149e980a37062f3001187 2024-11-16T10:24:16,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/606ceb7832094eb68a9d828122834b43 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/606ceb7832094eb68a9d828122834b43 2024-11-16T10:24:16,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/533350ee8d8b451e8e9a38ab9873979b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/533350ee8d8b451e8e9a38ab9873979b 2024-11-16T10:24:16,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/f8f13c7b51e34fc7ae49d5309355a28b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/f8f13c7b51e34fc7ae49d5309355a28b 2024-11-16T10:24:16,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/e7b52a00abfc423dbce919b951aa299d to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/e7b52a00abfc423dbce919b951aa299d 2024-11-16T10:24:16,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/cc349a464bdf423284cd4fda424feb0e to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/cc349a464bdf423284cd4fda424feb0e 2024-11-16T10:24:16,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/3a7c5c1ea7784149b72131dd01ef3b38 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/3a7c5c1ea7784149b72131dd01ef3b38 2024-11-16T10:24:16,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/007c6c5f05184c2f82f7b4ad611f67f0 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/007c6c5f05184c2f82f7b4ad611f67f0 2024-11-16T10:24:16,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/0fb5d05194a54cfc99c1144c30221d1b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/0fb5d05194a54cfc99c1144c30221d1b 2024-11-16T10:24:16,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/a9387214362447ddbae78595a2d178cd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/a9387214362447ddbae78595a2d178cd 2024-11-16T10:24:16,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/d17eab35c4f046508bece576327e7d5a to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/d17eab35c4f046508bece576327e7d5a 2024-11-16T10:24:16,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/724507c405cc47f6abea8d56cf1ba0b0 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/724507c405cc47f6abea8d56cf1ba0b0 2024-11-16T10:24:16,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/30b1f880edee40c9bc8c80bcb3fcaf50 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/30b1f880edee40c9bc8c80bcb3fcaf50 2024-11-16T10:24:16,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/0213d015c1f34893a35751c24f33308d to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/0213d015c1f34893a35751c24f33308d 2024-11-16T10:24:16,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/5ceeab731970406aae29b33e29e5ee3a to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/5ceeab731970406aae29b33e29e5ee3a 2024-11-16T10:24:16,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/9b8ae0408b1742f5be2789919025d7db to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/9b8ae0408b1742f5be2789919025d7db 2024-11-16T10:24:16,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/a950908d6f8747ec9fbe9d04d975fbc4 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/a950908d6f8747ec9fbe9d04d975fbc4 2024-11-16T10:24:16,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/3611204636e34c1089f3178bccbf01fe to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/3611204636e34c1089f3178bccbf01fe 2024-11-16T10:24:16,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/a5c2dd24d3ff4688a9e2e60053d61049 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/a5c2dd24d3ff4688a9e2e60053d61049 2024-11-16T10:24:16,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/1dfc9db6899d42d9920698f9f4cc7918 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/1dfc9db6899d42d9920698f9f4cc7918 2024-11-16T10:24:16,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/1fbae99c4d214a67be37be4105945ca5, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/dea9f6ac4edb4d2bb93580ba120a5b9f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/afe9112f2756453caa08e5201563daf2, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/048c9a6eb318480d937be3e5fc644622, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/d61e38faad0d437dbc19603f58291eed, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/a63b4374a72c42c6a66eab9c9c3986f3, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/eae8d4a34c6b40c8b77a2180c74327fd, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/0b10967bf22f41dbb8c677d6f67fd5c1, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/754d806cb6124bd58fe066b4a1641a9e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/d5995e3162194da5a53d4d049daefd0b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/62e726fdc5a14f709b23c2e9b3c627cc, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/ae3f0c266be04b539f148f1b0b3ae046, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/4fdec96e905e4d0ca1d64d0b59923f14, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/71743cbcebc447718c63d6ef15670b0a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/e196b86eff8744b1995a29a812ae5069, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/ff997d7e9f2b49089cf60aab618257f7, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/fff44d94818d4c00aa7e21d55a60f020, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/04cb683d56bb4b3aaea9609af1826d7b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/057dd860bb4d4f758e4e130c7780b6a7, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/b8c4eff5c7344b5892e22817096a04c8, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/9397539034664aa4bbad996f0d14bd6c, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/4567316d21df4e498e53add9e39861c9, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/ef634949411e4f0792ed60d685716beb, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/427daca9446d4107a26be36b6752283d, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/38ed5e63b78144959279121205348170, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/554a0ac19c9c4a3a8fad49a259a36a3f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/dd8c02fd184f45bd92b7a7cee80d1103, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/5c08bac83efa4e59bf25b53bfddc4dd0, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/33c50bf8a52c48d4844bd4aee91e0ed2, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/6f4554f932c14d2493ab1c6979ab2690, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/75f747d6cdc247e4aee572831fa0597f] to archive 2024-11-16T10:24:16,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:24:16,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/1fbae99c4d214a67be37be4105945ca5 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/1fbae99c4d214a67be37be4105945ca5 2024-11-16T10:24:16,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/dea9f6ac4edb4d2bb93580ba120a5b9f to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/dea9f6ac4edb4d2bb93580ba120a5b9f 2024-11-16T10:24:16,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/afe9112f2756453caa08e5201563daf2 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/afe9112f2756453caa08e5201563daf2 2024-11-16T10:24:16,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/048c9a6eb318480d937be3e5fc644622 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/048c9a6eb318480d937be3e5fc644622 2024-11-16T10:24:16,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/d61e38faad0d437dbc19603f58291eed to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/d61e38faad0d437dbc19603f58291eed 2024-11-16T10:24:16,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/a63b4374a72c42c6a66eab9c9c3986f3 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/a63b4374a72c42c6a66eab9c9c3986f3 2024-11-16T10:24:16,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/eae8d4a34c6b40c8b77a2180c74327fd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/eae8d4a34c6b40c8b77a2180c74327fd 2024-11-16T10:24:16,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/0b10967bf22f41dbb8c677d6f67fd5c1 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/0b10967bf22f41dbb8c677d6f67fd5c1 2024-11-16T10:24:16,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/754d806cb6124bd58fe066b4a1641a9e to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/754d806cb6124bd58fe066b4a1641a9e 2024-11-16T10:24:16,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/d5995e3162194da5a53d4d049daefd0b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/d5995e3162194da5a53d4d049daefd0b 2024-11-16T10:24:16,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/62e726fdc5a14f709b23c2e9b3c627cc to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/62e726fdc5a14f709b23c2e9b3c627cc 2024-11-16T10:24:16,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/ae3f0c266be04b539f148f1b0b3ae046 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/ae3f0c266be04b539f148f1b0b3ae046 2024-11-16T10:24:16,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/4fdec96e905e4d0ca1d64d0b59923f14 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/4fdec96e905e4d0ca1d64d0b59923f14 2024-11-16T10:24:16,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/71743cbcebc447718c63d6ef15670b0a to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/71743cbcebc447718c63d6ef15670b0a 2024-11-16T10:24:16,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/e196b86eff8744b1995a29a812ae5069 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/e196b86eff8744b1995a29a812ae5069 2024-11-16T10:24:16,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/ff997d7e9f2b49089cf60aab618257f7 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/ff997d7e9f2b49089cf60aab618257f7 2024-11-16T10:24:16,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/fff44d94818d4c00aa7e21d55a60f020 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/fff44d94818d4c00aa7e21d55a60f020 2024-11-16T10:24:16,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/04cb683d56bb4b3aaea9609af1826d7b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/04cb683d56bb4b3aaea9609af1826d7b 2024-11-16T10:24:16,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/057dd860bb4d4f758e4e130c7780b6a7 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/057dd860bb4d4f758e4e130c7780b6a7 2024-11-16T10:24:16,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/b8c4eff5c7344b5892e22817096a04c8 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/b8c4eff5c7344b5892e22817096a04c8 2024-11-16T10:24:16,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/9397539034664aa4bbad996f0d14bd6c to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/9397539034664aa4bbad996f0d14bd6c 2024-11-16T10:24:16,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/4567316d21df4e498e53add9e39861c9 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/4567316d21df4e498e53add9e39861c9 2024-11-16T10:24:16,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/ef634949411e4f0792ed60d685716beb to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/ef634949411e4f0792ed60d685716beb 2024-11-16T10:24:16,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/427daca9446d4107a26be36b6752283d to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/427daca9446d4107a26be36b6752283d 2024-11-16T10:24:16,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/38ed5e63b78144959279121205348170 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/38ed5e63b78144959279121205348170 2024-11-16T10:24:16,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/554a0ac19c9c4a3a8fad49a259a36a3f to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/554a0ac19c9c4a3a8fad49a259a36a3f 2024-11-16T10:24:16,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/dd8c02fd184f45bd92b7a7cee80d1103 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/dd8c02fd184f45bd92b7a7cee80d1103 2024-11-16T10:24:16,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/5c08bac83efa4e59bf25b53bfddc4dd0 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/5c08bac83efa4e59bf25b53bfddc4dd0 2024-11-16T10:24:16,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/33c50bf8a52c48d4844bd4aee91e0ed2 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/33c50bf8a52c48d4844bd4aee91e0ed2 2024-11-16T10:24:16,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/6f4554f932c14d2493ab1c6979ab2690 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/6f4554f932c14d2493ab1c6979ab2690 2024-11-16T10:24:16,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/75f747d6cdc247e4aee572831fa0597f to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/75f747d6cdc247e4aee572831fa0597f 2024-11-16T10:24:16,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/61216607784a47a5ace23c975309c00b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/ec523327315e4b1ba7afdc359e77291d, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/2fe18e4c7ffe4bc7971554dbdf144b88, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/4a0a8da7b27047b9aef5ccbffa56923c, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/3900fc6a2bc94519961def6b77ce1508, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/52362e1d518b47ca867a046d695e865f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/b86ca6c550ef468885aacce3b99995de, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/d2b144d39fd940b59548b4c562cb0ce8, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/43dcd99d485c4c62a1bb61f6ea92368e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/e88f65af4c67464597dbf547776939e2, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/c04676ff5c8f4531b41149cea438d9ff, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/891faf2231e647dd8d3168e19083e87b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/05bc4159766341a6b03c5678a7162f9a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/64b023defa2a4f34b853e3e05050f26a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/84a52682c3704594be3112bbc398ffa2, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/e9c930d074374255a9b12516ab62fc8c, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/b96336fe9b224503b5fd5eb6103b7e15, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/e12df7db30954c4a9b54b4210731588c, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/d7b089dc014849c7ad1d8c62de1f1111, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/e46269ee704444c8b4b96be08091e572, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/005bebc101734d0da2161fa29674a561, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/a25f9ddc0a4c49ea849d4bb444516c19, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/4b158d8efdba4f1b950e63b36c0a3e6f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/cd87a031c11c4335a322583944139e6e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/9ac1febc1ab042f0af828f2f28d583e0, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/72abd22a403c45ffa5bd2a9861bb8e05, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/5d068b499e7b4d2d9cd79693b6889950, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/2a608a767a574401b3ebf531e7554ded, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/b0027f143972466e95b15d742ebe5ada, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/b2025b4700b646ef9b38512471348e43, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/094bc01b5f65416cba32d3e7c91938df] to archive 2024-11-16T10:24:16,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:24:16,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/61216607784a47a5ace23c975309c00b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/61216607784a47a5ace23c975309c00b 2024-11-16T10:24:16,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/ec523327315e4b1ba7afdc359e77291d to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/ec523327315e4b1ba7afdc359e77291d 2024-11-16T10:24:16,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/2fe18e4c7ffe4bc7971554dbdf144b88 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/2fe18e4c7ffe4bc7971554dbdf144b88 2024-11-16T10:24:16,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/4a0a8da7b27047b9aef5ccbffa56923c to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/4a0a8da7b27047b9aef5ccbffa56923c 2024-11-16T10:24:16,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/3900fc6a2bc94519961def6b77ce1508 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/3900fc6a2bc94519961def6b77ce1508 2024-11-16T10:24:16,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/52362e1d518b47ca867a046d695e865f to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/52362e1d518b47ca867a046d695e865f 2024-11-16T10:24:16,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/b86ca6c550ef468885aacce3b99995de to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/b86ca6c550ef468885aacce3b99995de 2024-11-16T10:24:16,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/d2b144d39fd940b59548b4c562cb0ce8 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/d2b144d39fd940b59548b4c562cb0ce8 2024-11-16T10:24:16,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/43dcd99d485c4c62a1bb61f6ea92368e to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/43dcd99d485c4c62a1bb61f6ea92368e 2024-11-16T10:24:16,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/e88f65af4c67464597dbf547776939e2 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/e88f65af4c67464597dbf547776939e2 2024-11-16T10:24:16,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/c04676ff5c8f4531b41149cea438d9ff to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/c04676ff5c8f4531b41149cea438d9ff 2024-11-16T10:24:16,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/891faf2231e647dd8d3168e19083e87b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/891faf2231e647dd8d3168e19083e87b 2024-11-16T10:24:16,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/05bc4159766341a6b03c5678a7162f9a to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/05bc4159766341a6b03c5678a7162f9a 2024-11-16T10:24:16,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/64b023defa2a4f34b853e3e05050f26a to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/64b023defa2a4f34b853e3e05050f26a 2024-11-16T10:24:16,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/84a52682c3704594be3112bbc398ffa2 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/84a52682c3704594be3112bbc398ffa2 2024-11-16T10:24:16,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/e9c930d074374255a9b12516ab62fc8c to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/e9c930d074374255a9b12516ab62fc8c 2024-11-16T10:24:16,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/b96336fe9b224503b5fd5eb6103b7e15 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/b96336fe9b224503b5fd5eb6103b7e15 2024-11-16T10:24:16,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/e12df7db30954c4a9b54b4210731588c to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/e12df7db30954c4a9b54b4210731588c 2024-11-16T10:24:16,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/d7b089dc014849c7ad1d8c62de1f1111 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/d7b089dc014849c7ad1d8c62de1f1111 2024-11-16T10:24:16,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/e46269ee704444c8b4b96be08091e572 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/e46269ee704444c8b4b96be08091e572 2024-11-16T10:24:16,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/005bebc101734d0da2161fa29674a561 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/005bebc101734d0da2161fa29674a561 2024-11-16T10:24:16,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/a25f9ddc0a4c49ea849d4bb444516c19 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/a25f9ddc0a4c49ea849d4bb444516c19 2024-11-16T10:24:16,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/4b158d8efdba4f1b950e63b36c0a3e6f to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/4b158d8efdba4f1b950e63b36c0a3e6f 2024-11-16T10:24:16,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/cd87a031c11c4335a322583944139e6e to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/cd87a031c11c4335a322583944139e6e 2024-11-16T10:24:16,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/9ac1febc1ab042f0af828f2f28d583e0 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/9ac1febc1ab042f0af828f2f28d583e0 2024-11-16T10:24:16,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/72abd22a403c45ffa5bd2a9861bb8e05 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/72abd22a403c45ffa5bd2a9861bb8e05 2024-11-16T10:24:16,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/5d068b499e7b4d2d9cd79693b6889950 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/5d068b499e7b4d2d9cd79693b6889950 2024-11-16T10:24:16,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/2a608a767a574401b3ebf531e7554ded to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/2a608a767a574401b3ebf531e7554ded 2024-11-16T10:24:16,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/b0027f143972466e95b15d742ebe5ada to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/b0027f143972466e95b15d742ebe5ada 2024-11-16T10:24:16,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/b2025b4700b646ef9b38512471348e43 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/b2025b4700b646ef9b38512471348e43 2024-11-16T10:24:16,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/094bc01b5f65416cba32d3e7c91938df to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/094bc01b5f65416cba32d3e7c91938df 2024-11-16T10:24:16,715 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/recovered.edits/480.seqid, newMaxSeqId=480, maxSeqId=1 2024-11-16T10:24:16,715 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. 2024-11-16T10:24:16,716 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for e63c645d595c80b0e763eedbebd195b2: Waiting for close lock at 1731752655214Running coprocessor pre-close hooks at 1731752655214Disabling compacts and flushes for region at 1731752655214Disabling writes for close at 1731752655215 (+1 ms)Obtaining lock to block concurrent updates at 1731752655215Preparing flush snapshotting stores in e63c645d595c80b0e763eedbebd195b2 at 1731752655215Finished memstore snapshotting TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731752655216 (+1 ms)Flushing stores of TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2. at 1731752655217 (+1 ms)Flushing e63c645d595c80b0e763eedbebd195b2/A: creating writer at 1731752655217Flushing e63c645d595c80b0e763eedbebd195b2/A: appending metadata at 1731752655222 (+5 ms)Flushing e63c645d595c80b0e763eedbebd195b2/A: closing flushed file at 1731752655222Flushing e63c645d595c80b0e763eedbebd195b2/B: creating writer at 1731752655636 (+414 ms)Flushing e63c645d595c80b0e763eedbebd195b2/B: appending metadata at 1731752655639 (+3 ms)Flushing e63c645d595c80b0e763eedbebd195b2/B: closing flushed file at 1731752655639Flushing e63c645d595c80b0e763eedbebd195b2/C: creating writer at 1731752656064 (+425 ms)Flushing e63c645d595c80b0e763eedbebd195b2/C: appending metadata at 1731752656068 (+4 ms)Flushing e63c645d595c80b0e763eedbebd195b2/C: closing flushed file at 1731752656068Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6aa3ae28: reopening flushed file at 1731752656486 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5930334a: reopening flushed file at 1731752656493 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f034f45: reopening flushed file at 1731752656500 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for e63c645d595c80b0e763eedbebd195b2 in 1292ms, sequenceid=477, compaction requested=true at 1731752656507 (+7 ms)Writing region close event to WAL at 1731752656711 (+204 ms)Running coprocessor post-close hooks at 1731752656715 (+4 ms)Closed at 1731752656715 2024-11-16T10:24:16,718 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:24:16,718 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=e63c645d595c80b0e763eedbebd195b2, regionState=CLOSED 2024-11-16T10:24:16,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 because future has completed 2024-11-16T10:24:16,724 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-16T10:24:16,724 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure e63c645d595c80b0e763eedbebd195b2, server=b16ff38458c1,35873,1731752627506 in 1.6690 sec 2024-11-16T10:24:16,725 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-16T10:24:16,725 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e63c645d595c80b0e763eedbebd195b2, UNASSIGN in 1.6770 sec 2024-11-16T10:24:16,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-16T10:24:16,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6850 sec 2024-11-16T10:24:16,730 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752656730"}]},"ts":"1731752656730"} 2024-11-16T10:24:16,732 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T10:24:16,732 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T10:24:16,735 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7000 sec 2024-11-16T10:24:17,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T10:24:17,187 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:24:17,190 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T10:24:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:24:17,195 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:24:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-16T10:24:17,197 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:24:17,200 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:24:17,204 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A, FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B, FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C, FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/recovered.edits] 2024-11-16T10:24:17,207 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/6b7562162f494bc7a8508612d51d6dfa to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/6b7562162f494bc7a8508612d51d6dfa 2024-11-16T10:24:17,209 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/9c902d53fbbc4bbcb2c2862724273696 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/9c902d53fbbc4bbcb2c2862724273696 2024-11-16T10:24:17,210 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/b5b460020fe54bb5a7fdda74bc3529fa to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/A/b5b460020fe54bb5a7fdda74bc3529fa 2024-11-16T10:24:17,213 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/7cdf63fb1aeb4253bb9ec64401eed7f2 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/7cdf63fb1aeb4253bb9ec64401eed7f2 2024-11-16T10:24:17,214 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/e2d2e440dbd447d78ababf57f281f279 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/e2d2e440dbd447d78ababf57f281f279 2024-11-16T10:24:17,215 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/f5237245ec864f04b5f6fea811e3e6f5 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/B/f5237245ec864f04b5f6fea811e3e6f5 2024-11-16T10:24:17,218 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/0930a8ee7b904f33b587046c4caf3bc4 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/0930a8ee7b904f33b587046c4caf3bc4 2024-11-16T10:24:17,219 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/6f628be58d0f4b5cb896e6e2223d08ad to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/6f628be58d0f4b5cb896e6e2223d08ad 2024-11-16T10:24:17,220 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/ac7bbc1a01e040bcb18442a67e4d756b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/C/ac7bbc1a01e040bcb18442a67e4d756b 2024-11-16T10:24:17,222 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/recovered.edits/480.seqid to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2/recovered.edits/480.seqid 2024-11-16T10:24:17,223 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/e63c645d595c80b0e763eedbebd195b2 2024-11-16T10:24:17,223 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T10:24:17,227 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:24:17,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-16T10:24:17,236 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T10:24:17,266 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T10:24:17,267 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:24:17,267 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T10:24:17,268 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731752657267"}]},"ts":"9223372036854775807"} 2024-11-16T10:24:17,272 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T10:24:17,272 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e63c645d595c80b0e763eedbebd195b2, NAME => 'TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T10:24:17,272 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T10:24:17,273 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731752657272"}]},"ts":"9223372036854775807"} 2024-11-16T10:24:17,276 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T10:24:17,277 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:24:17,278 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 87 msec 2024-11-16T10:24:17,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-16T10:24:17,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T10:24:17,307 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:24:17,319 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=239 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;b16ff38458c1:35873-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1713545760_22 at /127.0.0.1:37672 [Waiting for operation #137] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107388350_22 at /127.0.0.1:59594 [Waiting for operation #267] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107388350_22 at /127.0.0.1:37280 [Waiting for operation #252] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=552 (was 396) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4359 (was 5039) 2024-11-16T10:24:17,327 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=552, ProcessCount=11, AvailableMemoryMB=4359 2024-11-16T10:24:17,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T10:24:17,329 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T10:24:17,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T10:24:17,331 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T10:24:17,331 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:17,331 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-16T10:24:17,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-16T10:24:17,332 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T10:24:17,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741939_1115 (size=959) 2024-11-16T10:24:17,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-16T10:24:17,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-16T10:24:17,742 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7 2024-11-16T10:24:17,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741940_1116 (size=53) 2024-11-16T10:24:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-16T10:24:18,155 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:24:18,155 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3ea6f99fb63837c16170eecaeea220bd, disabling compactions & flushes 2024-11-16T10:24:18,155 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:18,155 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:18,155 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. after waiting 0 ms 2024-11-16T10:24:18,155 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:18,155 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:18,156 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3ea6f99fb63837c16170eecaeea220bd: Waiting for close lock at 1731752658155Disabling compacts and flushes for region at 1731752658155Disabling writes for close at 1731752658155Writing region close event to WAL at 1731752658155Closed at 1731752658155 2024-11-16T10:24:18,159 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T10:24:18,160 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731752658159"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731752658159"}]},"ts":"1731752658159"} 2024-11-16T10:24:18,164 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T10:24:18,166 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T10:24:18,166 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752658166"}]},"ts":"1731752658166"} 2024-11-16T10:24:18,168 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T10:24:18,168 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ea6f99fb63837c16170eecaeea220bd, ASSIGN}] 2024-11-16T10:24:18,170 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ea6f99fb63837c16170eecaeea220bd, ASSIGN 2024-11-16T10:24:18,170 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ea6f99fb63837c16170eecaeea220bd, ASSIGN; state=OFFLINE, location=b16ff38458c1,35873,1731752627506; forceNewPlan=false, retain=false 2024-11-16T10:24:18,322 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=3ea6f99fb63837c16170eecaeea220bd, regionState=OPENING, regionLocation=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:18,327 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ea6f99fb63837c16170eecaeea220bd, ASSIGN because future has completed 2024-11-16T10:24:18,328 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506}] 2024-11-16T10:24:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-16T10:24:18,486 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:18,486 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} 2024-11-16T10:24:18,486 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:18,486 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:24:18,486 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:18,486 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:18,488 INFO [StoreOpener-3ea6f99fb63837c16170eecaeea220bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:18,490 INFO [StoreOpener-3ea6f99fb63837c16170eecaeea220bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ea6f99fb63837c16170eecaeea220bd columnFamilyName A 2024-11-16T10:24:18,490 DEBUG [StoreOpener-3ea6f99fb63837c16170eecaeea220bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:18,491 INFO [StoreOpener-3ea6f99fb63837c16170eecaeea220bd-1 {}] regionserver.HStore(327): Store=3ea6f99fb63837c16170eecaeea220bd/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:24:18,491 INFO [StoreOpener-3ea6f99fb63837c16170eecaeea220bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:18,492 INFO [StoreOpener-3ea6f99fb63837c16170eecaeea220bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ea6f99fb63837c16170eecaeea220bd columnFamilyName B 2024-11-16T10:24:18,492 DEBUG [StoreOpener-3ea6f99fb63837c16170eecaeea220bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:18,492 INFO [StoreOpener-3ea6f99fb63837c16170eecaeea220bd-1 {}] regionserver.HStore(327): Store=3ea6f99fb63837c16170eecaeea220bd/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:24:18,492 INFO [StoreOpener-3ea6f99fb63837c16170eecaeea220bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:18,493 INFO [StoreOpener-3ea6f99fb63837c16170eecaeea220bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ea6f99fb63837c16170eecaeea220bd columnFamilyName C 2024-11-16T10:24:18,493 DEBUG [StoreOpener-3ea6f99fb63837c16170eecaeea220bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:18,494 INFO [StoreOpener-3ea6f99fb63837c16170eecaeea220bd-1 {}] regionserver.HStore(327): Store=3ea6f99fb63837c16170eecaeea220bd/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:24:18,494 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:18,494 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:18,495 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:18,496 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:18,496 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:18,496 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T10:24:18,498 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:18,500 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T10:24:18,500 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 3ea6f99fb63837c16170eecaeea220bd; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72297665, jitterRate=0.07731916010379791}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T10:24:18,500 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:18,501 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 3ea6f99fb63837c16170eecaeea220bd: Running coprocessor pre-open hook at 1731752658487Writing region info on filesystem at 1731752658487Initializing all the Stores at 1731752658487Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752658487Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752658488 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752658488Cleaning up temporary data from old regions at 1731752658496 (+8 ms)Running coprocessor post-open hooks at 1731752658500 (+4 ms)Region opened successfully at 1731752658501 (+1 ms) 2024-11-16T10:24:18,502 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., pid=32, masterSystemTime=1731752658482 2024-11-16T10:24:18,504 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:18,504 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:18,504 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=3ea6f99fb63837c16170eecaeea220bd, regionState=OPEN, openSeqNum=2, regionLocation=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:18,506 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 because future has completed 2024-11-16T10:24:18,509 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-16T10:24:18,509 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 in 179 msec 2024-11-16T10:24:18,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-16T10:24:18,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ea6f99fb63837c16170eecaeea220bd, ASSIGN in 341 msec 2024-11-16T10:24:18,512 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T10:24:18,512 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752658512"}]},"ts":"1731752658512"} 2024-11-16T10:24:18,513 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T10:24:18,514 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T10:24:18,516 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1850 sec 2024-11-16T10:24:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-16T10:24:19,477 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:24:19,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2697ccde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:19,479 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:24:19,480 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:24:19,482 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:24:19,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:24:19,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:24:19,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4be5c052, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:19,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:24:19,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:24:19,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:19,489 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:24:19,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@635a54b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:19,490 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:24:19,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:24:19,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:24:19,492 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:24:19,494 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T10:24:19,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is b16ff38458c1,38809,1731752626047 2024-11-16T10:24:19,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@70e495a7 2024-11-16T10:24:19,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T10:24:19,496 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T10:24:19,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T10:24:19,501 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T10:24:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-16T10:24:19,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741941_1117 (size=995) 2024-11-16T10:24:19,923 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-16T10:24:19,923 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-16T10:24:19,931 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T10:24:19,937 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ea6f99fb63837c16170eecaeea220bd, REOPEN/MOVE}] 2024-11-16T10:24:19,938 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ea6f99fb63837c16170eecaeea220bd, REOPEN/MOVE 2024-11-16T10:24:19,939 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=3ea6f99fb63837c16170eecaeea220bd, regionState=CLOSING, regionLocation=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:19,941 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ea6f99fb63837c16170eecaeea220bd, REOPEN/MOVE because future has completed 2024-11-16T10:24:19,941 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T10:24:19,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506}] 2024-11-16T10:24:20,094 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:20,094 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T10:24:20,094 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 3ea6f99fb63837c16170eecaeea220bd, disabling compactions & flushes 2024-11-16T10:24:20,094 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:20,094 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:20,095 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. after waiting 0 ms 2024-11-16T10:24:20,095 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:20,099 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T10:24:20,100 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:20,100 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 3ea6f99fb63837c16170eecaeea220bd: Waiting for close lock at 1731752660094Running coprocessor pre-close hooks at 1731752660094Disabling compacts and flushes for region at 1731752660094Disabling writes for close at 1731752660095 (+1 ms)Writing region close event to WAL at 1731752660095Running coprocessor post-close hooks at 1731752660100 (+5 ms)Closed at 1731752660100 2024-11-16T10:24:20,100 WARN [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: 3ea6f99fb63837c16170eecaeea220bd to self. 2024-11-16T10:24:20,103 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:20,103 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=3ea6f99fb63837c16170eecaeea220bd, regionState=CLOSED 2024-11-16T10:24:20,105 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 because future has completed 2024-11-16T10:24:20,108 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-16T10:24:20,109 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 in 165 msec 2024-11-16T10:24:20,109 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ea6f99fb63837c16170eecaeea220bd, REOPEN/MOVE; state=CLOSED, location=b16ff38458c1,35873,1731752627506; forceNewPlan=false, retain=true 2024-11-16T10:24:20,259 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=3ea6f99fb63837c16170eecaeea220bd, regionState=OPENING, regionLocation=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:20,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ea6f99fb63837c16170eecaeea220bd, REOPEN/MOVE because future has completed 2024-11-16T10:24:20,261 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506}] 2024-11-16T10:24:20,417 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:20,417 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} 2024-11-16T10:24:20,418 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:20,418 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:24:20,418 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:20,418 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:20,422 INFO [StoreOpener-3ea6f99fb63837c16170eecaeea220bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:20,428 INFO [StoreOpener-3ea6f99fb63837c16170eecaeea220bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ea6f99fb63837c16170eecaeea220bd columnFamilyName A 2024-11-16T10:24:20,431 DEBUG [StoreOpener-3ea6f99fb63837c16170eecaeea220bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:20,431 INFO [StoreOpener-3ea6f99fb63837c16170eecaeea220bd-1 {}] regionserver.HStore(327): Store=3ea6f99fb63837c16170eecaeea220bd/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:24:20,432 INFO [StoreOpener-3ea6f99fb63837c16170eecaeea220bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:20,433 INFO [StoreOpener-3ea6f99fb63837c16170eecaeea220bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ea6f99fb63837c16170eecaeea220bd columnFamilyName B 2024-11-16T10:24:20,433 DEBUG [StoreOpener-3ea6f99fb63837c16170eecaeea220bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:20,434 INFO [StoreOpener-3ea6f99fb63837c16170eecaeea220bd-1 {}] regionserver.HStore(327): Store=3ea6f99fb63837c16170eecaeea220bd/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:24:20,434 INFO [StoreOpener-3ea6f99fb63837c16170eecaeea220bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:20,435 INFO [StoreOpener-3ea6f99fb63837c16170eecaeea220bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ea6f99fb63837c16170eecaeea220bd columnFamilyName C 2024-11-16T10:24:20,435 DEBUG [StoreOpener-3ea6f99fb63837c16170eecaeea220bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:20,435 INFO [StoreOpener-3ea6f99fb63837c16170eecaeea220bd-1 {}] regionserver.HStore(327): Store=3ea6f99fb63837c16170eecaeea220bd/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:24:20,435 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:20,436 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:20,441 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:20,442 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:20,442 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:20,443 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T10:24:20,444 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:20,445 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 3ea6f99fb63837c16170eecaeea220bd; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67978112, jitterRate=0.012952804565429688}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T10:24:20,445 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:20,447 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 3ea6f99fb63837c16170eecaeea220bd: Running coprocessor pre-open hook at 1731752660418Writing region info on filesystem at 1731752660418Initializing all the Stores at 1731752660419 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752660419Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752660420 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752660420Cleaning up temporary data from old regions at 1731752660442 (+22 ms)Running coprocessor post-open hooks at 1731752660445 (+3 ms)Region opened successfully at 1731752660447 (+2 ms) 2024-11-16T10:24:20,448 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., pid=37, masterSystemTime=1731752660414 2024-11-16T10:24:20,451 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=3ea6f99fb63837c16170eecaeea220bd, regionState=OPEN, openSeqNum=5, regionLocation=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:20,452 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:20,452 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:20,453 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 because future has completed 2024-11-16T10:24:20,457 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-16T10:24:20,457 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 in 194 msec 2024-11-16T10:24:20,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-16T10:24:20,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ea6f99fb63837c16170eecaeea220bd, REOPEN/MOVE in 520 msec 2024-11-16T10:24:20,466 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-16T10:24:20,467 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 533 msec 2024-11-16T10:24:20,471 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 965 msec 2024-11-16T10:24:20,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-16T10:24:20,473 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:24:20,474 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:20,474 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:24:20,474 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:24:20,475 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:24:20,475 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:24:20,475 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:24:20,475 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5298de4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:20,475 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:24:20,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:24:20,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:20,477 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:24:20,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db1b3b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:20,478 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:20,478 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:24:20,478 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:24:20,479 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:24:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:24:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:24:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555e7ba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:24:20,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:24:20,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:20,481 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:24:20,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54ff8d39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:20,482 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:20,482 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:24:20,482 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:24:20,483 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:24:20,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:24:20,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:24:20,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@151fabe4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:20,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:24:20,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:24:20,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:20,485 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:24:20,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19bf19ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:20,488 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:20,488 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:24:20,488 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:24:20,489 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:24:20,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:24:20,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:24:20,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@318de19f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:20,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:24:20,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:24:20,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:20,491 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:24:20,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@637b3c3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:20,492 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:20,492 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:24:20,493 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:24:20,493 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:24:20,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:24:20,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:24:20,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bacd749, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:20,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:24:20,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:24:20,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:20,495 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:24:20,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4c6c9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:20,497 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:20,497 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:24:20,497 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:24:20,498 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:24:20,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:24:20,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:24:20,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42293790, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:20,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:24:20,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:24:20,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:20,500 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:24:20,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47323d42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:20,504 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:20,504 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:24:20,505 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:24:20,506 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:24:20,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:24:20,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:24:20,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6803c853, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:20,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:24:20,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:24:20,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:20,507 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48020, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:24:20,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54c88dcf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:20,509 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:20,509 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:24:20,509 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:24:20,510 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:24:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:24:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:24:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@758782aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:24:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:24:20,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:20,511 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:24:20,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c602edf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:20,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123a694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:20,512 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:24:20,513 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:24:20,513 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:24:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:24:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:24:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47ec5c17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:24:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:24:20,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:20,516 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:24:20,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aff2db3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:20,517 DEBUG [Thread-601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:24:20,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:24:20,518 DEBUG [Thread-614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:24:20,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:24:20,519 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:24:20,519 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:24:20,519 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:24:20,520 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:24:20,522 DEBUG [Thread-605 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:24:20,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5] 2024-11-16T10:24:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5] 2024-11-16T10:24:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:24:20,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:24:20,524 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:24:20,525 DEBUG [Thread-608 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:24:20,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:24:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-16T10:24:20,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:24:20,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:24:20,527 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:24:20,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5] 2024-11-16T10:24:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T10:24:20,528 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:24:20,528 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:24:20,528 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:24:20,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5] 2024-11-16T10:24:20,532 DEBUG [Thread-597 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:24:20,532 DEBUG [Thread-610 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:24:20,532 DEBUG [Thread-603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:24:20,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:24:20,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:24:20,534 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:24:20,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:24:20,535 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:24:20,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:24:20,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:24:20,536 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59064, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:24:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5] 2024-11-16T10:24:20,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5] 2024-11-16T10:24:20,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:20,541 DEBUG [Thread-599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:24:20,541 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:24:20,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:24:20,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:24:20,542 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:24:20,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ea6f99fb63837c16170eecaeea220bd 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T10:24:20,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5] 2024-11-16T10:24:20,544 DEBUG [Thread-612 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:24:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:24:20,545 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:24:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:24:20,547 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:24:20,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5] 2024-11-16T10:24:20,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5] 2024-11-16T10:24:20,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752720586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:20,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:20,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:20,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752720590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:20,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:20,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:20,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59032 deadline: 1731752720591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:20,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:20,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b6899a112f724f62bd9b47ff771dcd29_3ea6f99fb63837c16170eecaeea220bd is 50, key is test_row_0/A:col10/1731752660533/Put/seqid=0 2024-11-16T10:24:20,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:20,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752720598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:20,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59064 deadline: 1731752720598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:20,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741942_1118 (size=12154) 2024-11-16T10:24:20,617 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:20,623 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b6899a112f724f62bd9b47ff771dcd29_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b6899a112f724f62bd9b47ff771dcd29_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:20,624 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/6f952ab716944e8bbd32522a3688f68c, store: [table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:20,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/6f952ab716944e8bbd32522a3688f68c is 175, key is test_row_0/A:col10/1731752660533/Put/seqid=0 2024-11-16T10:24:20,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T10:24:20,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741943_1119 (size=30955) 2024-11-16T10:24:20,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T10:24:20,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:20,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:20,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:20,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:20,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:20,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:20,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:20,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752720706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:20,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:20,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59032 deadline: 1731752720707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:20,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:20,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:20,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752720707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:20,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:20,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752720716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:20,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:20,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59064 deadline: 1731752720716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:20,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:20,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:20,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T10:24:20,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:20,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:20,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:20,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:20,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:20,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:20,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T10:24:20,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752720916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:20,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:20,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752720916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59032 deadline: 1731752720916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:20,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:20,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:20,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59064 deadline: 1731752720927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:20,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752720927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:20,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:20,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:20,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T10:24:20,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:20,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:20,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:20,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:20,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:20,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:21,051 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=18, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/6f952ab716944e8bbd32522a3688f68c 2024-11-16T10:24:21,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/93cf5096393c4bff8905931e9bfea196 is 50, key is test_row_0/B:col10/1731752660533/Put/seqid=0 2024-11-16T10:24:21,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741944_1120 (size=12001) 2024-11-16T10:24:21,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T10:24:21,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:21,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:21,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:21,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:21,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:21,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T10:24:21,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:21,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752721226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:21,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:21,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:21,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752721226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:21,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:21,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59032 deadline: 1731752721226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:21,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:21,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:21,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59064 deadline: 1731752721237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:21,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:21,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752721237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:21,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:21,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:21,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T10:24:21,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:21,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:21,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:21,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:21,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:21,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:21,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T10:24:21,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:21,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:21,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:21,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:21,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:21,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/93cf5096393c4bff8905931e9bfea196 2024-11-16T10:24:21,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/dce857a9156c42b198dd8000a4037503 is 50, key is test_row_0/C:col10/1731752660533/Put/seqid=0 2024-11-16T10:24:21,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741945_1121 (size=12001) 2024-11-16T10:24:21,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/dce857a9156c42b198dd8000a4037503 2024-11-16T10:24:21,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/6f952ab716944e8bbd32522a3688f68c as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/6f952ab716944e8bbd32522a3688f68c 2024-11-16T10:24:21,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T10:24:21,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/6f952ab716944e8bbd32522a3688f68c, entries=150, sequenceid=18, filesize=30.2 K 2024-11-16T10:24:21,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:21,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:21,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:21,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:21,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:21,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:21,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/93cf5096393c4bff8905931e9bfea196 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/93cf5096393c4bff8905931e9bfea196 2024-11-16T10:24:21,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/93cf5096393c4bff8905931e9bfea196, entries=150, sequenceid=18, filesize=11.7 K 2024-11-16T10:24:21,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/dce857a9156c42b198dd8000a4037503 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/dce857a9156c42b198dd8000a4037503 2024-11-16T10:24:21,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/dce857a9156c42b198dd8000a4037503, entries=150, sequenceid=18, filesize=11.7 K 2024-11-16T10:24:21,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 3ea6f99fb63837c16170eecaeea220bd in 1105ms, sequenceid=18, compaction requested=false 2024-11-16T10:24:21,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ea6f99fb63837c16170eecaeea220bd: 2024-11-16T10:24:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T10:24:21,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:21,744 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:24:21,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ea6f99fb63837c16170eecaeea220bd 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T10:24:21,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T10:24:21,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:21,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:21,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:21,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:21,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:21,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:21,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b75d881d6d5e480d8f5ebc24f0058280_3ea6f99fb63837c16170eecaeea220bd is 50, key is test_row_0/A:col10/1731752661742/Put/seqid=0 2024-11-16T10:24:21,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:21,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59064 deadline: 1731752721786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:21,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:21,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:21,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752721786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:21,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:21,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752721786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:21,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:21,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:21,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59032 deadline: 1731752721789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:21,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:21,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:21,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752721797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:21,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:21,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741946_1122 (size=12154) 2024-11-16T10:24:21,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59064 deadline: 1731752721906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:21,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:21,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:21,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752721906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:21,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:21,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59032 deadline: 1731752721906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:21,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:21,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:21,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:21,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752721907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:21,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752721938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:21,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T10:24:21,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:21,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:21,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:21,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:22,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T10:24:22,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:22,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:22,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:22,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:22,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:22,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:22,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:22,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752722117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:22,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59064 deadline: 1731752722118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:22,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:22,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:22,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:22,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59032 deadline: 1731752722118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:22,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752722147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:22,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:22,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752722147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:22,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:22,232 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:22,237 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b75d881d6d5e480d8f5ebc24f0058280_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b75d881d6d5e480d8f5ebc24f0058280_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:22,240 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/0b86f0b6f0d243acbe494edb124bef52, store: [table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:22,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/0b86f0b6f0d243acbe494edb124bef52 is 175, key is test_row_0/A:col10/1731752661742/Put/seqid=0 2024-11-16T10:24:22,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T10:24:22,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:22,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:22,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:22,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:22,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:22,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741947_1123 (size=30955) 2024-11-16T10:24:22,284 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T10:24:22,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T10:24:22,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:22,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:22,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:22,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:22,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:22,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59032 deadline: 1731752722427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:22,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59064 deadline: 1731752722427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:22,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:22,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752722427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:22,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:22,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:22,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:22,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752722457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752722457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:22,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T10:24:22,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:22,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:22,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:22,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:22,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:22,663 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=43, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/0b86f0b6f0d243acbe494edb124bef52 2024-11-16T10:24:22,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/2ae8618396c74a35ae80701ca0ada023 is 50, key is test_row_0/B:col10/1731752661742/Put/seqid=0 2024-11-16T10:24:22,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T10:24:22,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741948_1124 (size=12001) 2024-11-16T10:24:22,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T10:24:22,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:22,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:22,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:22,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:22,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:22,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T10:24:22,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:22,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:22,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:22,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:22,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:22,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752722937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:22,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:22,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59032 deadline: 1731752722937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:22,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59064 deadline: 1731752722937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:22,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:22,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752722976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:22,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752722976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:23,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T10:24:23,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:23,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:23,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:23,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:23,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:23,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/2ae8618396c74a35ae80701ca0ada023 2024-11-16T10:24:23,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/5e1c5e15cfe142358bfe4bf487f7142b is 50, key is test_row_0/C:col10/1731752661742/Put/seqid=0 2024-11-16T10:24:23,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741949_1125 (size=12001) 2024-11-16T10:24:23,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/5e1c5e15cfe142358bfe4bf487f7142b 2024-11-16T10:24:23,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/0b86f0b6f0d243acbe494edb124bef52 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/0b86f0b6f0d243acbe494edb124bef52 2024-11-16T10:24:23,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/0b86f0b6f0d243acbe494edb124bef52, entries=150, sequenceid=43, filesize=30.2 K 2024-11-16T10:24:23,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/2ae8618396c74a35ae80701ca0ada023 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/2ae8618396c74a35ae80701ca0ada023 2024-11-16T10:24:23,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/2ae8618396c74a35ae80701ca0ada023, entries=150, sequenceid=43, filesize=11.7 K 2024-11-16T10:24:23,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/5e1c5e15cfe142358bfe4bf487f7142b as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/5e1c5e15cfe142358bfe4bf487f7142b 2024-11-16T10:24:23,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/5e1c5e15cfe142358bfe4bf487f7142b, entries=150, sequenceid=43, filesize=11.7 K 2024-11-16T10:24:23,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 3ea6f99fb63837c16170eecaeea220bd in 1401ms, sequenceid=43, compaction requested=false 2024-11-16T10:24:23,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ea6f99fb63837c16170eecaeea220bd: 2024-11-16T10:24:23,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-16T10:24:23,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:23,171 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 3ea6f99fb63837c16170eecaeea220bd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T10:24:23,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165d8b80ca388b4f42a8c5bcf189769b5e_3ea6f99fb63837c16170eecaeea220bd is 50, key is test_row_0/A:col10/1731752661773/Put/seqid=0 2024-11-16T10:24:23,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741950_1126 (size=12154) 2024-11-16T10:24:23,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,191 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165d8b80ca388b4f42a8c5bcf189769b5e_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165d8b80ca388b4f42a8c5bcf189769b5e_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:23,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/b6722dfd23b9402e93f3dcdef234aa2d, store: [table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:23,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/b6722dfd23b9402e93f3dcdef234aa2d is 175, key is test_row_0/A:col10/1731752661773/Put/seqid=0 2024-11-16T10:24:23,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741951_1127 (size=30955) 2024-11-16T10:24:23,199 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/b6722dfd23b9402e93f3dcdef234aa2d 2024-11-16T10:24:23,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/9fea73f1f6f3482992a11958aaf14064 is 50, key is test_row_0/B:col10/1731752661773/Put/seqid=0 2024-11-16T10:24:23,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741952_1128 (size=12001) 2024-11-16T10:24:23,224 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/9fea73f1f6f3482992a11958aaf14064 2024-11-16T10:24:23,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/36bae6a323e34df79066d7016c436c22 is 50, key is test_row_0/C:col10/1731752661773/Put/seqid=0 2024-11-16T10:24:23,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741953_1129 (size=12001) 2024-11-16T10:24:23,642 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/36bae6a323e34df79066d7016c436c22 2024-11-16T10:24:23,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/b6722dfd23b9402e93f3dcdef234aa2d as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/b6722dfd23b9402e93f3dcdef234aa2d 2024-11-16T10:24:23,653 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/b6722dfd23b9402e93f3dcdef234aa2d, entries=150, sequenceid=54, filesize=30.2 K 2024-11-16T10:24:23,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/9fea73f1f6f3482992a11958aaf14064 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/9fea73f1f6f3482992a11958aaf14064 2024-11-16T10:24:23,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,661 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/9fea73f1f6f3482992a11958aaf14064, entries=150, sequenceid=54, filesize=11.7 K 2024-11-16T10:24:23,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/36bae6a323e34df79066d7016c436c22 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/36bae6a323e34df79066d7016c436c22 2024-11-16T10:24:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,671 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/36bae6a323e34df79066d7016c436c22, entries=150, sequenceid=54, filesize=11.7 K 2024-11-16T10:24:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,672 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for 3ea6f99fb63837c16170eecaeea220bd in 501ms, sequenceid=54, compaction requested=true 2024-11-16T10:24:23,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 3ea6f99fb63837c16170eecaeea220bd: 2024-11-16T10:24:23,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:23,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-16T10:24:23,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-16T10:24:23,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,676 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-16T10:24:23,676 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1450 sec 2024-11-16T10:24:23,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,679 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 3.1530 sec 2024-11-16T10:24:23,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,031 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:24:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ea6f99fb63837c16170eecaeea220bd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T10:24:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111604cf8b2dc5f24b70b50986f5d6bf5fdd_3ea6f99fb63837c16170eecaeea220bd is 50, key is test_row_0/A:col10/1731752664021/Put/seqid=0 2024-11-16T10:24:24,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:24,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752724089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:24,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:24,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:24,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752724089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:24,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:24,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741955_1131 (size=24358) 2024-11-16T10:24:24,104 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:24,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:24,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752724095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:24,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:24,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59064 deadline: 1731752724096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:24,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:24,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59032 deadline: 1731752724096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:24,112 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111604cf8b2dc5f24b70b50986f5d6bf5fdd_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111604cf8b2dc5f24b70b50986f5d6bf5fdd_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:24,113 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/c4e139c1b8e0402a83a8d266ae3ad5bd, store: [table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:24,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/c4e139c1b8e0402a83a8d266ae3ad5bd is 175, key is test_row_0/A:col10/1731752664021/Put/seqid=0 2024-11-16T10:24:24,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741954_1130 (size=73995) 2024-11-16T10:24:24,121 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=65, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/c4e139c1b8e0402a83a8d266ae3ad5bd 2024-11-16T10:24:24,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/826a2cbeb85a4fa78fbdf2edc1e258cc is 50, key is test_row_0/B:col10/1731752664021/Put/seqid=0 2024-11-16T10:24:24,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741956_1132 (size=12001) 2024-11-16T10:24:24,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:24,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:24,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752724206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:24,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752724207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:24,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:24,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:24,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:24,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59064 deadline: 1731752724217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:24,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:24,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:24,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752724217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:24,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:24,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:24,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59032 deadline: 1731752724217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:24,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:24,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:24,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752724416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:24,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:24,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:24,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752724426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:24,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:24,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:24,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752724426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:24,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:24,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:24,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59032 deadline: 1731752724427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:24,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:24,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59064 deadline: 1731752724426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:24,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:24,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/826a2cbeb85a4fa78fbdf2edc1e258cc 2024-11-16T10:24:24,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/eb324486126c49bd8333bcaf45b90621 is 50, key is test_row_0/C:col10/1731752664021/Put/seqid=0 2024-11-16T10:24:24,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741957_1133 (size=12001) 2024-11-16T10:24:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T10:24:24,709 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:24:24,709 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:24:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-16T10:24:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T10:24:24,713 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:24:24,714 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:24:24,714 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:24:24,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752724726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:24,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752724737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:24,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:24,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752724737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:24,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:24,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:24,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59032 deadline: 1731752724746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:24,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59064 deadline: 1731752724746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:24,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T10:24:24,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T10:24:24,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:24,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:24,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:24,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:24,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:25,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T10:24:25,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:25,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:25,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:25,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:25,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:25,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T10:24:25,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/eb324486126c49bd8333bcaf45b90621 2024-11-16T10:24:25,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/c4e139c1b8e0402a83a8d266ae3ad5bd as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/c4e139c1b8e0402a83a8d266ae3ad5bd 2024-11-16T10:24:25,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/c4e139c1b8e0402a83a8d266ae3ad5bd, entries=400, sequenceid=65, filesize=72.3 K 2024-11-16T10:24:25,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/826a2cbeb85a4fa78fbdf2edc1e258cc as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/826a2cbeb85a4fa78fbdf2edc1e258cc 2024-11-16T10:24:25,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/826a2cbeb85a4fa78fbdf2edc1e258cc, entries=150, sequenceid=65, filesize=11.7 K 2024-11-16T10:24:25,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/eb324486126c49bd8333bcaf45b90621 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/eb324486126c49bd8333bcaf45b90621 2024-11-16T10:24:25,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/eb324486126c49bd8333bcaf45b90621, entries=150, sequenceid=65, filesize=11.7 K 2024-11-16T10:24:25,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 3ea6f99fb63837c16170eecaeea220bd in 1040ms, sequenceid=65, compaction requested=true 2024-11-16T10:24:25,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ea6f99fb63837c16170eecaeea220bd: 2024-11-16T10:24:25,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ea6f99fb63837c16170eecaeea220bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:24:25,072 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:24:25,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:25,072 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:24:25,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ea6f99fb63837c16170eecaeea220bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:24:25,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:25,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ea6f99fb63837c16170eecaeea220bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:24:25,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:24:25,073 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:24:25,073 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): 3ea6f99fb63837c16170eecaeea220bd/B is initiating minor compaction (all files) 2024-11-16T10:24:25,074 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ea6f99fb63837c16170eecaeea220bd/B in TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:25,074 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/93cf5096393c4bff8905931e9bfea196, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/2ae8618396c74a35ae80701ca0ada023, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/9fea73f1f6f3482992a11958aaf14064, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/826a2cbeb85a4fa78fbdf2edc1e258cc] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp, totalSize=46.9 K 2024-11-16T10:24:25,074 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 166860 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:24:25,074 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): 3ea6f99fb63837c16170eecaeea220bd/A is initiating minor compaction (all files) 2024-11-16T10:24:25,074 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ea6f99fb63837c16170eecaeea220bd/A in TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:25,074 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/6f952ab716944e8bbd32522a3688f68c, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/0b86f0b6f0d243acbe494edb124bef52, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/b6722dfd23b9402e93f3dcdef234aa2d, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/c4e139c1b8e0402a83a8d266ae3ad5bd] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp, totalSize=162.9 K 2024-11-16T10:24:25,074 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:25,075 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. files: [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/6f952ab716944e8bbd32522a3688f68c, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/0b86f0b6f0d243acbe494edb124bef52, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/b6722dfd23b9402e93f3dcdef234aa2d, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/c4e139c1b8e0402a83a8d266ae3ad5bd] 2024-11-16T10:24:25,075 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 93cf5096393c4bff8905931e9bfea196, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1731752660533 2024-11-16T10:24:25,076 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f952ab716944e8bbd32522a3688f68c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1731752660533 2024-11-16T10:24:25,076 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ae8618396c74a35ae80701ca0ada023, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731752660586 2024-11-16T10:24:25,076 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b86f0b6f0d243acbe494edb124bef52, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731752660586 2024-11-16T10:24:25,076 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fea73f1f6f3482992a11958aaf14064, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731752661773 2024-11-16T10:24:25,077 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6722dfd23b9402e93f3dcdef234aa2d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731752661773 2024-11-16T10:24:25,077 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4e139c1b8e0402a83a8d266ae3ad5bd, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731752663962 2024-11-16T10:24:25,077 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 826a2cbeb85a4fa78fbdf2edc1e258cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731752664019 2024-11-16T10:24:25,097 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:25,107 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ea6f99fb63837c16170eecaeea220bd#B#compaction#115 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:25,108 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/ebac1cd030614e2788b2da0ae73a7167 is 50, key is test_row_0/B:col10/1731752664021/Put/seqid=0 2024-11-16T10:24:25,121 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411163f826d738527488b8cea2eb76cf2adc6_3ea6f99fb63837c16170eecaeea220bd store=[table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:25,127 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411163f826d738527488b8cea2eb76cf2adc6_3ea6f99fb63837c16170eecaeea220bd, store=[table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:25,127 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163f826d738527488b8cea2eb76cf2adc6_3ea6f99fb63837c16170eecaeea220bd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:25,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741959_1135 (size=4469) 2024-11-16T10:24:25,160 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ea6f99fb63837c16170eecaeea220bd#A#compaction#114 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:25,161 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/a0e6ef3b8a794453bda3d31357cb8634 is 175, key is test_row_0/A:col10/1731752664021/Put/seqid=0 2024-11-16T10:24:25,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741958_1134 (size=12139) 2024-11-16T10:24:25,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741960_1136 (size=31093) 2024-11-16T10:24:25,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T10:24:25,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:25,177 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 3ea6f99fb63837c16170eecaeea220bd 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-16T10:24:25,182 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/a0e6ef3b8a794453bda3d31357cb8634 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/a0e6ef3b8a794453bda3d31357cb8634 2024-11-16T10:24:25,190 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3ea6f99fb63837c16170eecaeea220bd/A of 3ea6f99fb63837c16170eecaeea220bd into a0e6ef3b8a794453bda3d31357cb8634(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:25,192 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ea6f99fb63837c16170eecaeea220bd: 2024-11-16T10:24:25,192 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., storeName=3ea6f99fb63837c16170eecaeea220bd/A, priority=12, startTime=1731752665071; duration=0sec 2024-11-16T10:24:25,192 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:24:25,193 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ea6f99fb63837c16170eecaeea220bd:A 2024-11-16T10:24:25,193 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:24:25,196 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:24:25,196 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): 3ea6f99fb63837c16170eecaeea220bd/C is initiating minor compaction (all files) 2024-11-16T10:24:25,196 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ea6f99fb63837c16170eecaeea220bd/C in TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:25,196 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/dce857a9156c42b198dd8000a4037503, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/5e1c5e15cfe142358bfe4bf487f7142b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/36bae6a323e34df79066d7016c436c22, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/eb324486126c49bd8333bcaf45b90621] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp, totalSize=46.9 K 2024-11-16T10:24:25,197 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting dce857a9156c42b198dd8000a4037503, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1731752660533 2024-11-16T10:24:25,197 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e1c5e15cfe142358bfe4bf487f7142b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731752660586 2024-11-16T10:24:25,198 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36bae6a323e34df79066d7016c436c22, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731752661773 2024-11-16T10:24:25,199 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb324486126c49bd8333bcaf45b90621, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731752664019 2024-11-16T10:24:25,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116408e7cd2260d4fbf80eaaeb1699dc179_3ea6f99fb63837c16170eecaeea220bd is 50, key is test_row_0/A:col10/1731752664087/Put/seqid=0 2024-11-16T10:24:25,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:25,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:25,246 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ea6f99fb63837c16170eecaeea220bd#C#compaction#117 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:25,247 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/7ee63e453b99417bb9cd6e1a2f7b5a01 is 50, key is test_row_0/C:col10/1731752664021/Put/seqid=0 2024-11-16T10:24:25,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741961_1137 (size=12154) 2024-11-16T10:24:25,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:25,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:25,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:25,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59032 deadline: 1731752725256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:25,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59064 deadline: 1731752725257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:25,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:25,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752725258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:25,266 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116408e7cd2260d4fbf80eaaeb1699dc179_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116408e7cd2260d4fbf80eaaeb1699dc179_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:25,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:25,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:25,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:25,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:25,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:25,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:25,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:25,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:25,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:25,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/48561b82da054794bdac6147693fcb08, store: [table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:25,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/48561b82da054794bdac6147693fcb08 is 175, key is test_row_0/A:col10/1731752664087/Put/seqid=0 2024-11-16T10:24:25,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:25,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752725266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:25,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:25,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752725268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:25,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:25,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:25,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:25,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:25,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:25,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:25,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741962_1138 (size=12139) 2024-11-16T10:24:25,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741963_1139 (size=30955) 2024-11-16T10:24:25,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T10:24:25,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:25,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752725376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:25,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:25,378 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:25,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:25,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752725376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:25,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:25,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:25,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:25,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:25,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752725376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:25,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:25,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:25,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:25,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:25,568 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/ebac1cd030614e2788b2da0ae73a7167 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/ebac1cd030614e2788b2da0ae73a7167 2024-11-16T10:24:25,575 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3ea6f99fb63837c16170eecaeea220bd/B of 3ea6f99fb63837c16170eecaeea220bd into ebac1cd030614e2788b2da0ae73a7167(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:25,575 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ea6f99fb63837c16170eecaeea220bd: 2024-11-16T10:24:25,575 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., storeName=3ea6f99fb63837c16170eecaeea220bd/B, priority=12, startTime=1731752665072; duration=0sec 2024-11-16T10:24:25,575 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:25,575 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ea6f99fb63837c16170eecaeea220bd:B 2024-11-16T10:24:25,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752725587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:25,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:25,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:25,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752725587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:25,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:25,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752725587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:25,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:25,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:25,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:25,682 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/7ee63e453b99417bb9cd6e1a2f7b5a01 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/7ee63e453b99417bb9cd6e1a2f7b5a01 2024-11-16T10:24:25,689 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3ea6f99fb63837c16170eecaeea220bd/C of 3ea6f99fb63837c16170eecaeea220bd into 7ee63e453b99417bb9cd6e1a2f7b5a01(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:25,689 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ea6f99fb63837c16170eecaeea220bd: 2024-11-16T10:24:25,690 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., storeName=3ea6f99fb63837c16170eecaeea220bd/C, priority=12, startTime=1731752665072; duration=0sec 2024-11-16T10:24:25,690 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:25,690 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ea6f99fb63837c16170eecaeea220bd:C 2024-11-16T10:24:25,695 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=91, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/48561b82da054794bdac6147693fcb08 2024-11-16T10:24:25,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/dd6f25ef5f5f45788c6e93f2d0e7fbf5 is 50, key is test_row_0/B:col10/1731752664087/Put/seqid=0 2024-11-16T10:24:25,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741964_1140 (size=12001) 2024-11-16T10:24:25,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T10:24:25,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:25,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752725897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:25,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:25,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:25,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:25,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:25,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752725897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:25,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752725897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:25,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:25,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:25,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:25,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:25,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:25,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:25,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:26,135 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/dd6f25ef5f5f45788c6e93f2d0e7fbf5 2024-11-16T10:24:26,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/b5a564824e2b49aa8aa80572d6d623cb is 50, key is test_row_0/C:col10/1731752664087/Put/seqid=0 2024-11-16T10:24:26,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741965_1141 (size=12001) 2024-11-16T10:24:26,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59064 deadline: 1731752726276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:26,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59032 deadline: 1731752726276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:26,277 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_2' in 3ea6f99fb63837c16170eecaeea220bd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:24:26,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:26,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:26,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:26,277 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_2' in 3ea6f99fb63837c16170eecaeea220bd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:24:26,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:26,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:26,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:26,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752726406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:26,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:26,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:26,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:26,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752726406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:26,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:26,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:26,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:26,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752726406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:26,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:26,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:26,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:26,617 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/b5a564824e2b49aa8aa80572d6d623cb 2024-11-16T10:24:26,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/48561b82da054794bdac6147693fcb08 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/48561b82da054794bdac6147693fcb08 2024-11-16T10:24:26,634 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/48561b82da054794bdac6147693fcb08, entries=150, sequenceid=91, filesize=30.2 K 2024-11-16T10:24:26,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/dd6f25ef5f5f45788c6e93f2d0e7fbf5 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/dd6f25ef5f5f45788c6e93f2d0e7fbf5 2024-11-16T10:24:26,642 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/dd6f25ef5f5f45788c6e93f2d0e7fbf5, entries=150, sequenceid=91, filesize=11.7 K 2024-11-16T10:24:26,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/b5a564824e2b49aa8aa80572d6d623cb as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/b5a564824e2b49aa8aa80572d6d623cb 2024-11-16T10:24:26,652 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/b5a564824e2b49aa8aa80572d6d623cb, entries=150, sequenceid=91, filesize=11.7 K 2024-11-16T10:24:26,653 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 3ea6f99fb63837c16170eecaeea220bd in 1476ms, sequenceid=91, compaction requested=false 2024-11-16T10:24:26,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 3ea6f99fb63837c16170eecaeea220bd: 2024-11-16T10:24:26,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:26,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-16T10:24:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-16T10:24:26,658 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-16T10:24:26,658 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9410 sec 2024-11-16T10:24:26,662 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.9510 sec 2024-11-16T10:24:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T10:24:26,857 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:24:26,857 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:24:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-16T10:24:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T10:24:26,859 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:24:26,860 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:24:26,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:24:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T10:24:27,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T10:24:27,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:27,012 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 3ea6f99fb63837c16170eecaeea220bd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T10:24:27,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161c767dd47f4e4b6eac9aaf534b6ebfa7_3ea6f99fb63837c16170eecaeea220bd is 50, key is test_row_0/A:col10/1731752665247/Put/seqid=0 2024-11-16T10:24:27,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741966_1142 (size=12154) 2024-11-16T10:24:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T10:24:27,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:27,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:27,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:27,434 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161c767dd47f4e4b6eac9aaf534b6ebfa7_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161c767dd47f4e4b6eac9aaf534b6ebfa7_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:27,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/93d63deeb84c4a1e8891776713969219, store: [table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:27,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/93d63deeb84c4a1e8891776713969219 is 175, key is test_row_0/A:col10/1731752665247/Put/seqid=0 2024-11-16T10:24:27,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741967_1143 (size=30955) 2024-11-16T10:24:27,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:27,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752727458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:27,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:27,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:27,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:27,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:27,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752727460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:27,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:27,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:27,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:27,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752727464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:27,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:27,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:27,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:27,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T10:24:27,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:27,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752727567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:27,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:27,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:27,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:27,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:27,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752727576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:27,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:27,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752727576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:27,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:27,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752727776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:27,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:27,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:27,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:27,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:27,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752727786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752727786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:27,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:27,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:27,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:27,851 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=105, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/93d63deeb84c4a1e8891776713969219 2024-11-16T10:24:27,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/fdca30d912dd4c55bc401451277ddbb0 is 50, key is test_row_0/B:col10/1731752665247/Put/seqid=0 2024-11-16T10:24:27,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741968_1144 (size=12001) 2024-11-16T10:24:27,865 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/fdca30d912dd4c55bc401451277ddbb0 2024-11-16T10:24:27,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/56272c26a0b141cc9928013718507b1e is 50, key is test_row_0/C:col10/1731752665247/Put/seqid=0 2024-11-16T10:24:27,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741969_1145 (size=12001) 2024-11-16T10:24:27,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T10:24:28,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752728086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:28,088 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:28,088 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:28,088 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:28,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752728096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:28,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:28,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752728096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:28,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:28,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:28,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:28,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:28,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:28,277 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/56272c26a0b141cc9928013718507b1e 2024-11-16T10:24:28,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/93d63deeb84c4a1e8891776713969219 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/93d63deeb84c4a1e8891776713969219 2024-11-16T10:24:28,288 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/93d63deeb84c4a1e8891776713969219, entries=150, sequenceid=105, filesize=30.2 K 2024-11-16T10:24:28,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/fdca30d912dd4c55bc401451277ddbb0 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/fdca30d912dd4c55bc401451277ddbb0 2024-11-16T10:24:28,294 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/fdca30d912dd4c55bc401451277ddbb0, entries=150, sequenceid=105, filesize=11.7 K 2024-11-16T10:24:28,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/56272c26a0b141cc9928013718507b1e as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/56272c26a0b141cc9928013718507b1e 2024-11-16T10:24:28,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:28,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:28,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59064 deadline: 1731752728296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:28,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59032 deadline: 1731752728297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:28,300 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_2' in 3ea6f99fb63837c16170eecaeea220bd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:24:28,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:28,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:28,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:28,301 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_2' in 3ea6f99fb63837c16170eecaeea220bd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:24:28,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:28,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:28,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:28,303 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/56272c26a0b141cc9928013718507b1e, entries=150, sequenceid=105, filesize=11.7 K 2024-11-16T10:24:28,304 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 3ea6f99fb63837c16170eecaeea220bd in 1292ms, sequenceid=105, compaction requested=true 2024-11-16T10:24:28,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 3ea6f99fb63837c16170eecaeea220bd: 2024-11-16T10:24:28,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:28,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-16T10:24:28,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-16T10:24:28,308 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-16T10:24:28,308 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4460 sec 2024-11-16T10:24:28,311 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.4520 sec 2024-11-16T10:24:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:28,598 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:24:28,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ea6f99fb63837c16170eecaeea220bd 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-16T10:24:28,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111660313e998a574a7ca1820ac16875ebe3_3ea6f99fb63837c16170eecaeea220bd is 50, key is test_row_0/A:col10/1731752668597/Put/seqid=0 2024-11-16T10:24:28,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752728615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:28,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752728616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:28,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752728616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:28,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:28,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:28,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:28,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:28,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:28,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:28,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:28,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:28,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:28,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741970_1146 (size=14694) 2024-11-16T10:24:28,627 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:28,631 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111660313e998a574a7ca1820ac16875ebe3_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111660313e998a574a7ca1820ac16875ebe3_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:28,632 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/e91918b706da41ab86b9627954463735, store: [table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:28,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/e91918b706da41ab86b9627954463735 is 175, key is test_row_0/A:col10/1731752668597/Put/seqid=0 2024-11-16T10:24:28,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741971_1147 (size=39649) 2024-11-16T10:24:28,638 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/e91918b706da41ab86b9627954463735 2024-11-16T10:24:28,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/e591290d60444334aee6641ec143917d is 50, key is test_row_0/B:col10/1731752668597/Put/seqid=0 2024-11-16T10:24:28,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741972_1148 (size=12101) 2024-11-16T10:24:28,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752728726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:28,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752728726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:28,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:28,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:28,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752728726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:28,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:28,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:28,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:28,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:28,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:28,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:28,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:28,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:28,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:28,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752728937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:28,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752728936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:28,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:28,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752728937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:29,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T10:24:29,007 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:24:29,007 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:24:29,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-16T10:24:29,011 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:24:29,011 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:24:29,012 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:24:29,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T10:24:29,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/e591290d60444334aee6641ec143917d 2024-11-16T10:24:29,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/048e510581504ba18eef71e767f03f0c is 50, key is test_row_0/C:col10/1731752668597/Put/seqid=0 2024-11-16T10:24:29,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741973_1149 (size=12101) 2024-11-16T10:24:29,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/048e510581504ba18eef71e767f03f0c 2024-11-16T10:24:29,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/e91918b706da41ab86b9627954463735 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/e91918b706da41ab86b9627954463735 2024-11-16T10:24:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T10:24:29,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/e91918b706da41ab86b9627954463735, entries=200, sequenceid=131, filesize=38.7 K 2024-11-16T10:24:29,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/e591290d60444334aee6641ec143917d as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/e591290d60444334aee6641ec143917d 2024-11-16T10:24:29,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/e591290d60444334aee6641ec143917d, entries=150, sequenceid=131, filesize=11.8 K 2024-11-16T10:24:29,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/048e510581504ba18eef71e767f03f0c as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/048e510581504ba18eef71e767f03f0c 2024-11-16T10:24:29,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/048e510581504ba18eef71e767f03f0c, entries=150, sequenceid=131, filesize=11.8 K 2024-11-16T10:24:29,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 3ea6f99fb63837c16170eecaeea220bd in 538ms, sequenceid=131, compaction requested=true 2024-11-16T10:24:29,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ea6f99fb63837c16170eecaeea220bd: 2024-11-16T10:24:29,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ea6f99fb63837c16170eecaeea220bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:24:29,137 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:24:29,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:29,137 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:24:29,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ea6f99fb63837c16170eecaeea220bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:24:29,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:29,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ea6f99fb63837c16170eecaeea220bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:24:29,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:24:29,138 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:24:29,138 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): 3ea6f99fb63837c16170eecaeea220bd/B is initiating minor compaction (all files) 2024-11-16T10:24:29,138 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ea6f99fb63837c16170eecaeea220bd/B in TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:29,139 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/ebac1cd030614e2788b2da0ae73a7167, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/dd6f25ef5f5f45788c6e93f2d0e7fbf5, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/fdca30d912dd4c55bc401451277ddbb0, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/e591290d60444334aee6641ec143917d] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp, totalSize=47.1 K 2024-11-16T10:24:29,139 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132652 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:24:29,139 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): 3ea6f99fb63837c16170eecaeea220bd/A is initiating minor compaction (all files) 2024-11-16T10:24:29,139 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ea6f99fb63837c16170eecaeea220bd/A in TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:29,139 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/a0e6ef3b8a794453bda3d31357cb8634, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/48561b82da054794bdac6147693fcb08, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/93d63deeb84c4a1e8891776713969219, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/e91918b706da41ab86b9627954463735] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp, totalSize=129.5 K 2024-11-16T10:24:29,139 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:29,139 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. files: [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/a0e6ef3b8a794453bda3d31357cb8634, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/48561b82da054794bdac6147693fcb08, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/93d63deeb84c4a1e8891776713969219, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/e91918b706da41ab86b9627954463735] 2024-11-16T10:24:29,140 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting ebac1cd030614e2788b2da0ae73a7167, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731752664019 2024-11-16T10:24:29,140 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0e6ef3b8a794453bda3d31357cb8634, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731752664019 2024-11-16T10:24:29,140 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting dd6f25ef5f5f45788c6e93f2d0e7fbf5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731752664087 2024-11-16T10:24:29,141 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48561b82da054794bdac6147693fcb08, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731752664087 2024-11-16T10:24:29,141 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting fdca30d912dd4c55bc401451277ddbb0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731752665247 2024-11-16T10:24:29,141 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93d63deeb84c4a1e8891776713969219, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731752665247 2024-11-16T10:24:29,141 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting e591290d60444334aee6641ec143917d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731752667456 2024-11-16T10:24:29,142 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting e91918b706da41ab86b9627954463735, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731752667456 2024-11-16T10:24:29,155 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:29,163 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ea6f99fb63837c16170eecaeea220bd#B#compaction#127 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:29,164 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/032ba778854c45019f051c62963362ad is 50, key is test_row_0/B:col10/1731752668597/Put/seqid=0 2024-11-16T10:24:29,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-16T10:24:29,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:29,166 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 3ea6f99fb63837c16170eecaeea220bd 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-16T10:24:29,171 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411169c1a98c9253340ce9cd917f109b87562_3ea6f99fb63837c16170eecaeea220bd store=[table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:29,174 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411169c1a98c9253340ce9cd917f109b87562_3ea6f99fb63837c16170eecaeea220bd, store=[table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:29,174 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169c1a98c9253340ce9cd917f109b87562_3ea6f99fb63837c16170eecaeea220bd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:29,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741974_1150 (size=12375) 2024-11-16T10:24:29,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741975_1151 (size=4469) 2024-11-16T10:24:29,206 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ea6f99fb63837c16170eecaeea220bd#A#compaction#126 average throughput is 0.48 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:29,207 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/812f1f3e77bb444d8cf4b55a1488b5cb is 175, key is test_row_0/A:col10/1731752668597/Put/seqid=0 2024-11-16T10:24:29,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c4811a352adc40b3881884f03e73266d_3ea6f99fb63837c16170eecaeea220bd is 50, key is test_row_0/A:col10/1731752668615/Put/seqid=0 2024-11-16T10:24:29,211 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/032ba778854c45019f051c62963362ad as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/032ba778854c45019f051c62963362ad 2024-11-16T10:24:29,220 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3ea6f99fb63837c16170eecaeea220bd/B of 3ea6f99fb63837c16170eecaeea220bd into 032ba778854c45019f051c62963362ad(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:29,221 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ea6f99fb63837c16170eecaeea220bd: 2024-11-16T10:24:29,221 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., storeName=3ea6f99fb63837c16170eecaeea220bd/B, priority=12, startTime=1731752669137; duration=0sec 2024-11-16T10:24:29,221 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:24:29,221 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ea6f99fb63837c16170eecaeea220bd:B 2024-11-16T10:24:29,221 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:24:29,223 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:24:29,223 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): 3ea6f99fb63837c16170eecaeea220bd/C is initiating minor compaction (all files) 2024-11-16T10:24:29,223 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ea6f99fb63837c16170eecaeea220bd/C in TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:29,223 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/7ee63e453b99417bb9cd6e1a2f7b5a01, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/b5a564824e2b49aa8aa80572d6d623cb, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/56272c26a0b141cc9928013718507b1e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/048e510581504ba18eef71e767f03f0c] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp, totalSize=47.1 K 2024-11-16T10:24:29,223 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ee63e453b99417bb9cd6e1a2f7b5a01, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731752664019 2024-11-16T10:24:29,224 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting b5a564824e2b49aa8aa80572d6d623cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731752664087 2024-11-16T10:24:29,226 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 56272c26a0b141cc9928013718507b1e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731752665247 2024-11-16T10:24:29,227 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 048e510581504ba18eef71e767f03f0c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731752667456 2024-11-16T10:24:29,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741976_1152 (size=31329) 2024-11-16T10:24:29,243 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/812f1f3e77bb444d8cf4b55a1488b5cb as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/812f1f3e77bb444d8cf4b55a1488b5cb 2024-11-16T10:24:29,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:29,251 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3ea6f99fb63837c16170eecaeea220bd/A of 3ea6f99fb63837c16170eecaeea220bd into 812f1f3e77bb444d8cf4b55a1488b5cb(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:29,251 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ea6f99fb63837c16170eecaeea220bd: 2024-11-16T10:24:29,251 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., storeName=3ea6f99fb63837c16170eecaeea220bd/A, priority=12, startTime=1731752669136; duration=0sec 2024-11-16T10:24:29,251 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:29,251 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ea6f99fb63837c16170eecaeea220bd:A 2024-11-16T10:24:29,259 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ea6f99fb63837c16170eecaeea220bd#C#compaction#129 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:29,259 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/f3f5a923f7e24768ac89df17ec3dd173 is 50, key is test_row_0/C:col10/1731752668597/Put/seqid=0 2024-11-16T10:24:29,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741977_1153 (size=12304) 2024-11-16T10:24:29,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741978_1154 (size=12375) 2024-11-16T10:24:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T10:24:29,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752729321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:29,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:29,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:29,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:29,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752729331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:29,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752729334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:29,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:29,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:29,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:29,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:29,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:29,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:29,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752729437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:29,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:29,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:29,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:29,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752729437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:29,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:29,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:29,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:29,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752729446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:29,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:29,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:29,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T10:24:29,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:29,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:29,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752729648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:29,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752729648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:29,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:29,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:29,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:29,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:29,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:29,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:29,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:29,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752729657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:29,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:29,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:29,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:29,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:29,669 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c4811a352adc40b3881884f03e73266d_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c4811a352adc40b3881884f03e73266d_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:29,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/ab8fb9106e6747b0bab94dc31413db10, store: [table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:29,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/ab8fb9106e6747b0bab94dc31413db10 is 175, key is test_row_0/A:col10/1731752668615/Put/seqid=0 2024-11-16T10:24:29,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741979_1155 (size=31105) 2024-11-16T10:24:29,699 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/f3f5a923f7e24768ac89df17ec3dd173 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/f3f5a923f7e24768ac89df17ec3dd173 2024-11-16T10:24:29,704 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3ea6f99fb63837c16170eecaeea220bd/C of 3ea6f99fb63837c16170eecaeea220bd into f3f5a923f7e24768ac89df17ec3dd173(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:29,704 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ea6f99fb63837c16170eecaeea220bd: 2024-11-16T10:24:29,704 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., storeName=3ea6f99fb63837c16170eecaeea220bd/C, priority=12, startTime=1731752669137; duration=0sec 2024-11-16T10:24:29,704 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:29,704 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ea6f99fb63837c16170eecaeea220bd:C 2024-11-16T10:24:29,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:29,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:29,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752729966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:29,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752729966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:29,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:29,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:29,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:29,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:29,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:29,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752729966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:29,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:29,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:29,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:29,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:29,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:30,077 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=141, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/ab8fb9106e6747b0bab94dc31413db10 2024-11-16T10:24:30,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/8291850e3f524b818c1d21d06e97af24 is 50, key is test_row_0/B:col10/1731752668615/Put/seqid=0 2024-11-16T10:24:30,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741980_1156 (size=12151) 2024-11-16T10:24:30,126 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/8291850e3f524b818c1d21d06e97af24 2024-11-16T10:24:30,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/4a45d7d0a84149b097b8a2b653a6d82b is 50, key is test_row_0/C:col10/1731752668615/Put/seqid=0 2024-11-16T10:24:30,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741981_1157 (size=12151) 2024-11-16T10:24:30,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T10:24:30,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:30,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752730477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:30,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:30,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752730477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:30,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:30,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:30,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752730477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:30,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:30,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:30,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:30,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:30,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:30,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:30,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:30,543 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/4a45d7d0a84149b097b8a2b653a6d82b 2024-11-16T10:24:30,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/ab8fb9106e6747b0bab94dc31413db10 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/ab8fb9106e6747b0bab94dc31413db10 2024-11-16T10:24:30,558 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/ab8fb9106e6747b0bab94dc31413db10, entries=150, sequenceid=141, filesize=30.4 K 2024-11-16T10:24:30,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/8291850e3f524b818c1d21d06e97af24 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/8291850e3f524b818c1d21d06e97af24 2024-11-16T10:24:30,567 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/8291850e3f524b818c1d21d06e97af24, entries=150, sequenceid=141, filesize=11.9 K 2024-11-16T10:24:30,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/4a45d7d0a84149b097b8a2b653a6d82b as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/4a45d7d0a84149b097b8a2b653a6d82b 2024-11-16T10:24:30,575 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/4a45d7d0a84149b097b8a2b653a6d82b, entries=150, sequenceid=141, filesize=11.9 K 2024-11-16T10:24:30,576 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 3ea6f99fb63837c16170eecaeea220bd in 1410ms, sequenceid=141, compaction requested=false 2024-11-16T10:24:30,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 3ea6f99fb63837c16170eecaeea220bd: 2024-11-16T10:24:30,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:30,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-16T10:24:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-16T10:24:30,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-16T10:24:30,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5670 sec 2024-11-16T10:24:30,582 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.5740 sec 2024-11-16T10:24:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T10:24:31,157 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:24:31,157 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:24:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-16T10:24:31,159 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:24:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T10:24:31,160 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:24:31,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:24:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T10:24:31,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T10:24:31,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:31,312 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 3ea6f99fb63837c16170eecaeea220bd 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-16T10:24:31,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c4e70fa6468a4207b5c59ad363ce0d5a_3ea6f99fb63837c16170eecaeea220bd is 50, key is test_row_0/A:col10/1731752669291/Put/seqid=0 2024-11-16T10:24:31,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741982_1158 (size=12304) 2024-11-16T10:24:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T10:24:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:31,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:31,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752731506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:31,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752731506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:31,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:31,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:31,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:31,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752731508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:31,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:31,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:31,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:31,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:31,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:31,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:31,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752731616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:31,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752731616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:31,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752731616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:31,724 INFO [master/b16ff38458c1:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-16T10:24:31,724 INFO [master/b16ff38458c1:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-16T10:24:31,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:31,732 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c4e70fa6468a4207b5c59ad363ce0d5a_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c4e70fa6468a4207b5c59ad363ce0d5a_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:31,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/44548865e7c84cda984a44f2c379696c, store: [table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:31,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/44548865e7c84cda984a44f2c379696c is 175, key is test_row_0/A:col10/1731752669291/Put/seqid=0 2024-11-16T10:24:31,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741983_1159 (size=31105) 2024-11-16T10:24:31,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T10:24:31,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:31,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752731826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:31,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:31,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:31,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:31,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:31,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752731826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:31,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:31,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752731826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:31,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:31,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:31,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:31,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:31,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:31,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:32,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:32,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752732136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:32,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:32,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:32,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:32,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752732136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:32,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:32,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:32,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:32,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:32,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:32,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752732136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:32,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:32,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:32,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:32,139 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/44548865e7c84cda984a44f2c379696c 2024-11-16T10:24:32,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/cb5a722a09cb45a2a9691baf50fb46a4 is 50, key is test_row_0/B:col10/1731752669291/Put/seqid=0 2024-11-16T10:24:32,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741984_1160 (size=12151) 2024-11-16T10:24:32,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T10:24:32,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:32,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59032 deadline: 1731752732317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:32,319 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_2' in 3ea6f99fb63837c16170eecaeea220bd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:24:32,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:32,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:32,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:32,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:32,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59064 deadline: 1731752732336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:32,337 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_2' in 3ea6f99fb63837c16170eecaeea220bd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:24:32,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:32,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:32,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:32,552 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/cb5a722a09cb45a2a9691baf50fb46a4 2024-11-16T10:24:32,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/7ea8bb3cfe0b4b3baaf0b4d24787bcd8 is 50, key is test_row_0/C:col10/1731752669291/Put/seqid=0 2024-11-16T10:24:32,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741985_1161 (size=12151) 2024-11-16T10:24:32,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:32,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752732646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:32,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:32,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:32,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752732646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:32,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:32,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:32,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:32,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:32,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:32,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752732646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:32,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:32,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:32,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:32,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:32,965 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/7ea8bb3cfe0b4b3baaf0b4d24787bcd8 2024-11-16T10:24:32,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/44548865e7c84cda984a44f2c379696c as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/44548865e7c84cda984a44f2c379696c 2024-11-16T10:24:32,974 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/44548865e7c84cda984a44f2c379696c, entries=150, sequenceid=171, filesize=30.4 K 2024-11-16T10:24:32,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-16T10:24:32,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/cb5a722a09cb45a2a9691baf50fb46a4 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/cb5a722a09cb45a2a9691baf50fb46a4 2024-11-16T10:24:32,981 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/cb5a722a09cb45a2a9691baf50fb46a4, entries=150, sequenceid=171, filesize=11.9 K 2024-11-16T10:24:32,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/7ea8bb3cfe0b4b3baaf0b4d24787bcd8 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/7ea8bb3cfe0b4b3baaf0b4d24787bcd8 2024-11-16T10:24:32,986 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/7ea8bb3cfe0b4b3baaf0b4d24787bcd8, entries=150, sequenceid=171, filesize=11.9 K 2024-11-16T10:24:32,987 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 3ea6f99fb63837c16170eecaeea220bd in 1675ms, sequenceid=171, compaction requested=true 2024-11-16T10:24:32,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 3ea6f99fb63837c16170eecaeea220bd: 2024-11-16T10:24:32,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:32,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-16T10:24:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-16T10:24:32,990 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-16T10:24:32,990 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8280 sec 2024-11-16T10:24:32,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.8330 sec 2024-11-16T10:24:33,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T10:24:33,307 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:24:33,307 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:24:33,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-16T10:24:33,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T10:24:33,309 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:24:33,310 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:24:33,310 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:24:33,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T10:24:33,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-16T10:24:33,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:33,462 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 3ea6f99fb63837c16170eecaeea220bd 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-16T10:24:33,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116eafacd4c026645dfb1ad97fd09cd4ca9_3ea6f99fb63837c16170eecaeea220bd is 50, key is test_row_0/A:col10/1731752671500/Put/seqid=0 2024-11-16T10:24:33,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741986_1162 (size=12304) 2024-11-16T10:24:33,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T10:24:33,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:33,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:33,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:33,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752733697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:33,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:33,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:33,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:33,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:33,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752733698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:33,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:33,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752733698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:33,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:33,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:33,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:33,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:33,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:33,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:33,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:33,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752733806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:33,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:33,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752733808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:33,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:33,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752733808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:33,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:33,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:33,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:33,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:33,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:33,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:33,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:33,876 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116eafacd4c026645dfb1ad97fd09cd4ca9_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116eafacd4c026645dfb1ad97fd09cd4ca9_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:33,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/31cb37d772b04a629bcdd99d3f9f6ca9, store: [table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:33,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/31cb37d772b04a629bcdd99d3f9f6ca9 is 175, key is test_row_0/A:col10/1731752671500/Put/seqid=0 2024-11-16T10:24:33,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741987_1163 (size=31105) 2024-11-16T10:24:33,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T10:24:34,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:34,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752734016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:34,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:34,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:34,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:34,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:34,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752734016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:34,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:34,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:34,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:34,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:34,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752734026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:34,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:34,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:34,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:34,282 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=180, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/31cb37d772b04a629bcdd99d3f9f6ca9 2024-11-16T10:24:34,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/f8607285b10b43379ad522feb16feb95 is 50, key is test_row_0/B:col10/1731752671500/Put/seqid=0 2024-11-16T10:24:34,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741988_1164 (size=12151) 2024-11-16T10:24:34,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:34,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752734326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:34,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:34,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:34,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:34,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:34,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752734326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:34,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:34,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:34,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:34,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:34,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752734336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:34,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:34,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:34,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:34,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T10:24:34,706 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/f8607285b10b43379ad522feb16feb95 2024-11-16T10:24:34,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/14446f32cb454cce8bf96ffdfe7da3e1 is 50, key is test_row_0/C:col10/1731752671500/Put/seqid=0 2024-11-16T10:24:34,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741989_1165 (size=12151) 2024-11-16T10:24:34,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752734836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:34,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:34,838 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:34,838 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752734836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:34,838 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:34,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:34,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:34,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:34,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752734846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:34,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:34,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:34,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:35,117 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/14446f32cb454cce8bf96ffdfe7da3e1 2024-11-16T10:24:35,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/31cb37d772b04a629bcdd99d3f9f6ca9 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/31cb37d772b04a629bcdd99d3f9f6ca9 2024-11-16T10:24:35,126 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/31cb37d772b04a629bcdd99d3f9f6ca9, entries=150, sequenceid=180, filesize=30.4 K 2024-11-16T10:24:35,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/f8607285b10b43379ad522feb16feb95 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/f8607285b10b43379ad522feb16feb95 2024-11-16T10:24:35,131 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/f8607285b10b43379ad522feb16feb95, entries=150, sequenceid=180, filesize=11.9 K 2024-11-16T10:24:35,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/14446f32cb454cce8bf96ffdfe7da3e1 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/14446f32cb454cce8bf96ffdfe7da3e1 2024-11-16T10:24:35,137 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/14446f32cb454cce8bf96ffdfe7da3e1, entries=150, sequenceid=180, filesize=11.9 K 2024-11-16T10:24:35,138 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=167.72 KB/171750 for 3ea6f99fb63837c16170eecaeea220bd in 1675ms, sequenceid=180, compaction requested=true 2024-11-16T10:24:35,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 3ea6f99fb63837c16170eecaeea220bd: 2024-11-16T10:24:35,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:35,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-16T10:24:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-16T10:24:35,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-16T10:24:35,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8290 sec 2024-11-16T10:24:35,142 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.8340 sec 2024-11-16T10:24:35,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T10:24:35,457 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:24:35,457 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:24:35,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-16T10:24:35,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T10:24:35,458 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:24:35,459 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:24:35,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:24:35,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T10:24:35,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T10:24:35,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:35,611 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 3ea6f99fb63837c16170eecaeea220bd 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-16T10:24:35,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163622bf5ea8684e939038c12090e5c62b_3ea6f99fb63837c16170eecaeea220bd is 50, key is test_row_0/A:col10/1731752673696/Put/seqid=0 2024-11-16T10:24:35,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741990_1166 (size=12304) 2024-11-16T10:24:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T10:24:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:35,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:35,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752735852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:35,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:35,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752735856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:35,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:35,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:35,858 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_2' in 3ea6f99fb63837c16170eecaeea220bd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:24:35,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:35,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:35,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:35,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752735857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:35,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:35,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:35,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:35,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:35,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752735968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752735968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:35,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:35,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:35,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:35,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:35,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:35,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:36,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:36,030 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163622bf5ea8684e939038c12090e5c62b_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163622bf5ea8684e939038c12090e5c62b_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:36,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/cbe4334d0fd249f6af2a3c5169330893, store: [table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:36,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/cbe4334d0fd249f6af2a3c5169330893 is 175, key is test_row_0/A:col10/1731752673696/Put/seqid=0 2024-11-16T10:24:36,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741991_1167 (size=31105) 2024-11-16T10:24:36,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T10:24:36,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:36,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752736176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:36,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:36,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752736176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:36,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:36,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:36,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:36,442 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/cbe4334d0fd249f6af2a3c5169330893 2024-11-16T10:24:36,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/060ea2614e6b457a87d10ce488894023 is 50, key is test_row_0/B:col10/1731752673696/Put/seqid=0 2024-11-16T10:24:36,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741992_1168 (size=12151) 2024-11-16T10:24:36,461 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/060ea2614e6b457a87d10ce488894023 2024-11-16T10:24:36,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/af6b06f2c988471a9bc53156253e95eb is 50, key is test_row_0/C:col10/1731752673696/Put/seqid=0 2024-11-16T10:24:36,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741993_1169 (size=12151) 2024-11-16T10:24:36,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752736486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:36,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752736486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:36,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:36,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:36,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:36,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:36,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:36,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T10:24:36,875 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/af6b06f2c988471a9bc53156253e95eb 2024-11-16T10:24:36,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/cbe4334d0fd249f6af2a3c5169330893 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/cbe4334d0fd249f6af2a3c5169330893 2024-11-16T10:24:36,884 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/cbe4334d0fd249f6af2a3c5169330893, entries=150, sequenceid=208, filesize=30.4 K 2024-11-16T10:24:36,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/060ea2614e6b457a87d10ce488894023 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/060ea2614e6b457a87d10ce488894023 2024-11-16T10:24:36,889 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/060ea2614e6b457a87d10ce488894023, entries=150, sequenceid=208, filesize=11.9 K 2024-11-16T10:24:36,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/af6b06f2c988471a9bc53156253e95eb as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/af6b06f2c988471a9bc53156253e95eb 2024-11-16T10:24:36,894 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/af6b06f2c988471a9bc53156253e95eb, entries=150, sequenceid=208, filesize=11.9 K 2024-11-16T10:24:36,895 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 3ea6f99fb63837c16170eecaeea220bd in 1284ms, sequenceid=208, compaction requested=true 2024-11-16T10:24:36,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 3ea6f99fb63837c16170eecaeea220bd: 2024-11-16T10:24:36,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:36,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-16T10:24:36,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-16T10:24:36,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-16T10:24:36,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4370 sec 2024-11-16T10:24:36,899 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.4410 sec 2024-11-16T10:24:37,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:37,012 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:24:37,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ea6f99fb63837c16170eecaeea220bd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T10:24:37,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111636b48292d49a4b3292867082252dc1cf_3ea6f99fb63837c16170eecaeea220bd is 50, key is test_row_0/A:col10/1731752677011/Put/seqid=0 2024-11-16T10:24:37,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741994_1170 (size=17284) 2024-11-16T10:24:37,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:37,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752737061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:37,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:37,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:37,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:37,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:37,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752737064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:37,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:37,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:37,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:37,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752737166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:37,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752737176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:37,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:37,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:37,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:37,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752737376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:37,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:37,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752737386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:37,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:37,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:37,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:37,436 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:37,440 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111636b48292d49a4b3292867082252dc1cf_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111636b48292d49a4b3292867082252dc1cf_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:37,440 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/211af02ac7844702921ba85cc989faaf, store: [table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:37,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/211af02ac7844702921ba85cc989faaf is 175, key is test_row_0/A:col10/1731752677011/Put/seqid=0 2024-11-16T10:24:37,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741995_1171 (size=48389) 2024-11-16T10:24:37,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T10:24:37,606 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:24:37,607 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:24:37,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-16T10:24:37,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T10:24:37,608 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:24:37,609 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:24:37,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:24:37,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:37,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752737686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:37,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:37,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:37,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:37,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:37,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752737696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:37,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:37,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:37,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:37,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T10:24:37,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T10:24:37,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:37,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:37,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:37,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:37,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:37,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:37,845 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=219, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/211af02ac7844702921ba85cc989faaf 2024-11-16T10:24:37,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/f821f952ba9146fd8d9057419d55eb26 is 50, key is test_row_0/B:col10/1731752677011/Put/seqid=0 2024-11-16T10:24:37,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741996_1172 (size=12151) 2024-11-16T10:24:37,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59082 deadline: 1731752737886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:37,890 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_2' in 3ea6f99fb63837c16170eecaeea220bd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:24:37,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:37,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:37,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:37,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T10:24:37,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:37,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:37,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:37,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:37,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:37,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:37,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T10:24:38,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T10:24:38,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:38,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:38,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:38,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:38,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:38,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:38,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752738196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:38,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:38,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:38,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:38,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:38,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752738207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:38,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:38,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:38,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:38,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T10:24:38,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:38,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:38,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:38,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:38,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:38,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:38,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T10:24:38,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/f821f952ba9146fd8d9057419d55eb26 2024-11-16T10:24:38,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/b9e51a94acb640d794febb9c587bc6ae is 50, key is test_row_0/C:col10/1731752677011/Put/seqid=0 2024-11-16T10:24:38,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741997_1173 (size=12151) 2024-11-16T10:24:38,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/b9e51a94acb640d794febb9c587bc6ae 2024-11-16T10:24:38,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/211af02ac7844702921ba85cc989faaf as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/211af02ac7844702921ba85cc989faaf 2024-11-16T10:24:38,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/211af02ac7844702921ba85cc989faaf, entries=250, sequenceid=219, filesize=47.3 K 2024-11-16T10:24:38,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/f821f952ba9146fd8d9057419d55eb26 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/f821f952ba9146fd8d9057419d55eb26 2024-11-16T10:24:38,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/f821f952ba9146fd8d9057419d55eb26, entries=150, sequenceid=219, filesize=11.9 K 2024-11-16T10:24:38,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/b9e51a94acb640d794febb9c587bc6ae as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/b9e51a94acb640d794febb9c587bc6ae 2024-11-16T10:24:38,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/b9e51a94acb640d794febb9c587bc6ae, entries=150, sequenceid=219, filesize=11.9 K 2024-11-16T10:24:38,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 3ea6f99fb63837c16170eecaeea220bd in 1299ms, sequenceid=219, compaction requested=true 2024-11-16T10:24:38,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ea6f99fb63837c16170eecaeea220bd: 2024-11-16T10:24:38,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ea6f99fb63837c16170eecaeea220bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:24:38,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:38,312 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-16T10:24:38,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ea6f99fb63837c16170eecaeea220bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:24:38,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:38,312 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-16T10:24:38,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ea6f99fb63837c16170eecaeea220bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:24:38,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:24:38,315 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 204138 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-16T10:24:38,315 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): 3ea6f99fb63837c16170eecaeea220bd/A is initiating minor compaction (all files) 2024-11-16T10:24:38,315 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ea6f99fb63837c16170eecaeea220bd/A in TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:38,315 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/812f1f3e77bb444d8cf4b55a1488b5cb, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/ab8fb9106e6747b0bab94dc31413db10, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/44548865e7c84cda984a44f2c379696c, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/31cb37d772b04a629bcdd99d3f9f6ca9, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/cbe4334d0fd249f6af2a3c5169330893, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/211af02ac7844702921ba85cc989faaf] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp, totalSize=199.4 K 2024-11-16T10:24:38,316 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:38,316 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. files: [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/812f1f3e77bb444d8cf4b55a1488b5cb, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/ab8fb9106e6747b0bab94dc31413db10, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/44548865e7c84cda984a44f2c379696c, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/31cb37d772b04a629bcdd99d3f9f6ca9, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/cbe4334d0fd249f6af2a3c5169330893, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/211af02ac7844702921ba85cc989faaf] 2024-11-16T10:24:38,316 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 812f1f3e77bb444d8cf4b55a1488b5cb, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731752667456 2024-11-16T10:24:38,316 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73130 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-16T10:24:38,316 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): 3ea6f99fb63837c16170eecaeea220bd/B is initiating minor compaction (all files) 2024-11-16T10:24:38,316 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ea6f99fb63837c16170eecaeea220bd/B in TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:38,316 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab8fb9106e6747b0bab94dc31413db10, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731752668608 2024-11-16T10:24:38,316 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/032ba778854c45019f051c62963362ad, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/8291850e3f524b818c1d21d06e97af24, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/cb5a722a09cb45a2a9691baf50fb46a4, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/f8607285b10b43379ad522feb16feb95, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/060ea2614e6b457a87d10ce488894023, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/f821f952ba9146fd8d9057419d55eb26] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp, totalSize=71.4 K 2024-11-16T10:24:38,317 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44548865e7c84cda984a44f2c379696c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731752669279 2024-11-16T10:24:38,317 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 032ba778854c45019f051c62963362ad, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731752667456 2024-11-16T10:24:38,317 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31cb37d772b04a629bcdd99d3f9f6ca9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731752671500 2024-11-16T10:24:38,317 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 8291850e3f524b818c1d21d06e97af24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731752668608 2024-11-16T10:24:38,318 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbe4334d0fd249f6af2a3c5169330893, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731752673695 2024-11-16T10:24:38,318 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting cb5a722a09cb45a2a9691baf50fb46a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731752669279 2024-11-16T10:24:38,318 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 211af02ac7844702921ba85cc989faaf, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731752675848 2024-11-16T10:24:38,318 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting f8607285b10b43379ad522feb16feb95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731752671500 2024-11-16T10:24:38,318 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 060ea2614e6b457a87d10ce488894023, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731752673695 2024-11-16T10:24:38,320 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting f821f952ba9146fd8d9057419d55eb26, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731752675848 2024-11-16T10:24:38,334 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:38,337 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ea6f99fb63837c16170eecaeea220bd#B#compaction#144 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:38,338 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/02ab6a714c8748d9b56ea54299b3fa57 is 50, key is test_row_0/B:col10/1731752677011/Put/seqid=0 2024-11-16T10:24:38,345 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116783f347096d7467e9e4f4cfe888dee56_3ea6f99fb63837c16170eecaeea220bd store=[table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:38,349 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116783f347096d7467e9e4f4cfe888dee56_3ea6f99fb63837c16170eecaeea220bd, store=[table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:38,349 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116783f347096d7467e9e4f4cfe888dee56_3ea6f99fb63837c16170eecaeea220bd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:38,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741998_1174 (size=12629) 2024-11-16T10:24:38,367 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/02ab6a714c8748d9b56ea54299b3fa57 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/02ab6a714c8748d9b56ea54299b3fa57 2024-11-16T10:24:38,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T10:24:38,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:38,372 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 3ea6f99fb63837c16170eecaeea220bd 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T10:24:38,373 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 3ea6f99fb63837c16170eecaeea220bd/B of 3ea6f99fb63837c16170eecaeea220bd into 02ab6a714c8748d9b56ea54299b3fa57(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:38,373 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ea6f99fb63837c16170eecaeea220bd: 2024-11-16T10:24:38,373 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., storeName=3ea6f99fb63837c16170eecaeea220bd/B, priority=10, startTime=1731752678312; duration=0sec 2024-11-16T10:24:38,373 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:24:38,373 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ea6f99fb63837c16170eecaeea220bd:B 2024-11-16T10:24:38,373 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-16T10:24:38,377 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73130 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-16T10:24:38,377 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): 3ea6f99fb63837c16170eecaeea220bd/C is initiating minor compaction (all files) 2024-11-16T10:24:38,377 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ea6f99fb63837c16170eecaeea220bd/C in TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:38,377 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/f3f5a923f7e24768ac89df17ec3dd173, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/4a45d7d0a84149b097b8a2b653a6d82b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/7ea8bb3cfe0b4b3baaf0b4d24787bcd8, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/14446f32cb454cce8bf96ffdfe7da3e1, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/af6b06f2c988471a9bc53156253e95eb, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/b9e51a94acb640d794febb9c587bc6ae] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp, totalSize=71.4 K 2024-11-16T10:24:38,378 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting f3f5a923f7e24768ac89df17ec3dd173, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731752667456 2024-11-16T10:24:38,378 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a45d7d0a84149b097b8a2b653a6d82b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731752668608 2024-11-16T10:24:38,378 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ea8bb3cfe0b4b3baaf0b4d24787bcd8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731752669279 2024-11-16T10:24:38,379 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 14446f32cb454cce8bf96ffdfe7da3e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731752671500 2024-11-16T10:24:38,379 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting af6b06f2c988471a9bc53156253e95eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731752673695 2024-11-16T10:24:38,379 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting b9e51a94acb640d794febb9c587bc6ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731752675848 2024-11-16T10:24:38,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741999_1175 (size=4469) 2024-11-16T10:24:38,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c8cda2f6b76d4f5896d784d577552d01_3ea6f99fb63837c16170eecaeea220bd is 50, key is test_row_0/A:col10/1731752677051/Put/seqid=0 2024-11-16T10:24:38,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742000_1176 (size=12304) 2024-11-16T10:24:38,400 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ea6f99fb63837c16170eecaeea220bd#C#compaction#147 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:38,400 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/3f788fceaebe463abc7d7816e603b44e is 50, key is test_row_0/C:col10/1731752677011/Put/seqid=0 2024-11-16T10:24:38,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742001_1177 (size=12629) 2024-11-16T10:24:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T10:24:38,785 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ea6f99fb63837c16170eecaeea220bd#A#compaction#145 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:38,786 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/424b7512b80a4c45a2b3beb30423a609 is 175, key is test_row_0/A:col10/1731752677011/Put/seqid=0 2024-11-16T10:24:38,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742002_1178 (size=31583) 2024-11-16T10:24:38,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:38,801 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c8cda2f6b76d4f5896d784d577552d01_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c8cda2f6b76d4f5896d784d577552d01_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:38,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/a5b3c69132054a8694617c5dfc030672, store: [table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:38,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/a5b3c69132054a8694617c5dfc030672 is 175, key is test_row_0/A:col10/1731752677051/Put/seqid=0 2024-11-16T10:24:38,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742003_1179 (size=31105) 2024-11-16T10:24:38,810 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/3f788fceaebe463abc7d7816e603b44e as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/3f788fceaebe463abc7d7816e603b44e 2024-11-16T10:24:38,815 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 3ea6f99fb63837c16170eecaeea220bd/C of 3ea6f99fb63837c16170eecaeea220bd into 3f788fceaebe463abc7d7816e603b44e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:38,815 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ea6f99fb63837c16170eecaeea220bd: 2024-11-16T10:24:38,815 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., storeName=3ea6f99fb63837c16170eecaeea220bd/C, priority=10, startTime=1731752678312; duration=0sec 2024-11-16T10:24:38,815 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:38,815 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ea6f99fb63837c16170eecaeea220bd:C 2024-11-16T10:24:39,195 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/424b7512b80a4c45a2b3beb30423a609 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/424b7512b80a4c45a2b3beb30423a609 2024-11-16T10:24:39,200 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 3ea6f99fb63837c16170eecaeea220bd/A of 3ea6f99fb63837c16170eecaeea220bd into 424b7512b80a4c45a2b3beb30423a609(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:39,200 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ea6f99fb63837c16170eecaeea220bd: 2024-11-16T10:24:39,200 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., storeName=3ea6f99fb63837c16170eecaeea220bd/A, priority=10, startTime=1731752678312; duration=0sec 2024-11-16T10:24:39,200 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:39,200 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ea6f99fb63837c16170eecaeea220bd:A 2024-11-16T10:24:39,206 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=245, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/a5b3c69132054a8694617c5dfc030672 2024-11-16T10:24:39,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:39,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:39,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/267c0402ebb64963b28eb83f8be9ef67 is 50, key is test_row_0/B:col10/1731752677051/Put/seqid=0 2024-11-16T10:24:39,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752739223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:39,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:39,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:39,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:39,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752739227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:39,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:39,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:39,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:39,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742004_1180 (size=12151) 2024-11-16T10:24:39,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:39,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752739337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:39,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:39,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752739337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:39,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:39,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:39,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:39,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:39,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:39,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:39,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:39,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752739546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:39,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:39,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:39,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752739546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:39,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:39,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:39,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:39,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:39,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:39,637 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/267c0402ebb64963b28eb83f8be9ef67 2024-11-16T10:24:39,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/7dcf13ac5094483b9ea2b4a39e4de552 is 50, key is test_row_0/C:col10/1731752677051/Put/seqid=0 2024-11-16T10:24:39,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742005_1181 (size=12151) 2024-11-16T10:24:39,648 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/7dcf13ac5094483b9ea2b4a39e4de552 2024-11-16T10:24:39,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/a5b3c69132054a8694617c5dfc030672 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/a5b3c69132054a8694617c5dfc030672 2024-11-16T10:24:39,656 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/a5b3c69132054a8694617c5dfc030672, entries=150, sequenceid=245, filesize=30.4 K 2024-11-16T10:24:39,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/267c0402ebb64963b28eb83f8be9ef67 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/267c0402ebb64963b28eb83f8be9ef67 2024-11-16T10:24:39,661 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/267c0402ebb64963b28eb83f8be9ef67, entries=150, sequenceid=245, filesize=11.9 K 2024-11-16T10:24:39,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/7dcf13ac5094483b9ea2b4a39e4de552 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/7dcf13ac5094483b9ea2b4a39e4de552 2024-11-16T10:24:39,669 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/7dcf13ac5094483b9ea2b4a39e4de552, entries=150, sequenceid=245, filesize=11.9 K 2024-11-16T10:24:39,670 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 3ea6f99fb63837c16170eecaeea220bd in 1298ms, sequenceid=245, compaction requested=false 2024-11-16T10:24:39,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 3ea6f99fb63837c16170eecaeea220bd: 2024-11-16T10:24:39,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:39,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-16T10:24:39,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-16T10:24:39,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-16T10:24:39,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0630 sec 2024-11-16T10:24:39,678 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.0700 sec 2024-11-16T10:24:39,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T10:24:39,756 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:24:39,757 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:24:39,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-16T10:24:39,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T10:24:39,760 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:24:39,761 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:24:39,761 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:24:39,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:39,858 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:24:39,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ea6f99fb63837c16170eecaeea220bd 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T10:24:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T10:24:39,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fa08956d5e7848e0ab50b4f0ad868611_3ea6f99fb63837c16170eecaeea220bd is 50, key is test_row_0/A:col10/1731752679221/Put/seqid=0 2024-11-16T10:24:39,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742006_1182 (size=17434) 2024-11-16T10:24:39,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752739904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:39,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:39,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:39,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752739905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:39,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:39,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:39,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:39,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:39,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-16T10:24:39,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:39,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:39,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:39,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:39,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:40,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:40,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752740016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752740016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:40,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:40,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:40,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:40,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:40,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:40,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:40,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-16T10:24:40,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:40,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:40,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:40,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:40,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T10:24:40,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-16T10:24:40,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:40,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:40,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:40,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:40,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:40,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:40,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752740226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:40,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:40,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752740226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:40,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:40,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:40,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:40,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:40,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:40,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:40,274 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:40,278 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fa08956d5e7848e0ab50b4f0ad868611_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fa08956d5e7848e0ab50b4f0ad868611_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:40,279 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/c961270271694cc990057d65c1d9504a, store: [table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:40,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/c961270271694cc990057d65c1d9504a is 175, key is test_row_0/A:col10/1731752679221/Put/seqid=0 2024-11-16T10:24:40,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742007_1183 (size=48539) 2024-11-16T10:24:40,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-16T10:24:40,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:40,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:40,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:40,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:40,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T10:24:40,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-16T10:24:40,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:40,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:40,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:40,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:40,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:40,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:40,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:40,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752740536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:40,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:40,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752740536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:40,543 INFO [Thread-608 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-608. 2024-11-16T10:24:40,543 DEBUG [Thread-608 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:24:40,543 DEBUG [Thread-608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:40,543 DEBUG [Thread-608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:40,543 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:24:40,543 INFO [Thread-612 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-612. 2024-11-16T10:24:40,543 DEBUG [Thread-612 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:24:40,543 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:40,543 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:40,544 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:24:40,546 INFO [Thread-610 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-610. 2024-11-16T10:24:40,546 DEBUG [Thread-610 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:24:40,546 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:40,546 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:40,546 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:24:40,549 INFO [Thread-614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-614. 2024-11-16T10:24:40,549 DEBUG [Thread-614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:24:40,549 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:40,550 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:40,550 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:24:40,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-16T10:24:40,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:40,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:40,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:40,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:40,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:40,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:40,685 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=260, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/c961270271694cc990057d65c1d9504a 2024-11-16T10:24:40,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/b4103c799c504dd18495185d25c59802 is 50, key is test_row_0/B:col10/1731752679221/Put/seqid=0 2024-11-16T10:24:40,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742008_1184 (size=12251) 2024-11-16T10:24:40,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-16T10:24:40,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:40,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:40,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:40,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:40,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:40,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T10:24:40,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-16T10:24:40,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:40,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:40,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:40,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:40,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:40,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:41,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:41,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:41,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59006 deadline: 1731752741047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:41,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59072 deadline: 1731752741048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:41,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:41,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:41,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:41,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:41,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:41,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:24:41,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/b4103c799c504dd18495185d25c59802 2024-11-16T10:24:41,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/7e0551885a914b998f6c654a43af128a is 50, key is test_row_0/C:col10/1731752679221/Put/seqid=0 2024-11-16T10:24:41,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742009_1185 (size=12251) 2024-11-16T10:24:41,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-16T10:24:41,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:41,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:41,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:41,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:41,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:41,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:41,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-16T10:24:41,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:41,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:41,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:41,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:41,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:41,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:41,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-16T10:24:41,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:41,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:41,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:41,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:41,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:41,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:41,521 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/7e0551885a914b998f6c654a43af128a 2024-11-16T10:24:41,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/c961270271694cc990057d65c1d9504a as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/c961270271694cc990057d65c1d9504a 2024-11-16T10:24:41,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/c961270271694cc990057d65c1d9504a, entries=250, sequenceid=260, filesize=47.4 K 2024-11-16T10:24:41,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/b4103c799c504dd18495185d25c59802 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/b4103c799c504dd18495185d25c59802 2024-11-16T10:24:41,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/b4103c799c504dd18495185d25c59802, entries=150, sequenceid=260, filesize=12.0 K 2024-11-16T10:24:41,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/7e0551885a914b998f6c654a43af128a as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/7e0551885a914b998f6c654a43af128a 2024-11-16T10:24:41,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/7e0551885a914b998f6c654a43af128a, entries=150, sequenceid=260, filesize=12.0 K 2024-11-16T10:24:41,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 3ea6f99fb63837c16170eecaeea220bd in 1681ms, sequenceid=260, compaction requested=true 2024-11-16T10:24:41,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ea6f99fb63837c16170eecaeea220bd: 2024-11-16T10:24:41,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ea6f99fb63837c16170eecaeea220bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:24:41,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:41,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ea6f99fb63837c16170eecaeea220bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:24:41,540 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:24:41,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:41,540 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:24:41,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ea6f99fb63837c16170eecaeea220bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:24:41,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:24:41,541 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111227 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:24:41,541 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37031 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:24:41,541 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): 3ea6f99fb63837c16170eecaeea220bd/A is initiating minor compaction (all files) 2024-11-16T10:24:41,541 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): 3ea6f99fb63837c16170eecaeea220bd/B is initiating minor compaction (all files) 2024-11-16T10:24:41,541 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ea6f99fb63837c16170eecaeea220bd/A in TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:41,541 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ea6f99fb63837c16170eecaeea220bd/B in TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:41,541 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/424b7512b80a4c45a2b3beb30423a609, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/a5b3c69132054a8694617c5dfc030672, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/c961270271694cc990057d65c1d9504a] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp, totalSize=108.6 K 2024-11-16T10:24:41,541 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/02ab6a714c8748d9b56ea54299b3fa57, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/267c0402ebb64963b28eb83f8be9ef67, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/b4103c799c504dd18495185d25c59802] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp, totalSize=36.2 K 2024-11-16T10:24:41,541 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:41,541 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. files: [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/424b7512b80a4c45a2b3beb30423a609, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/a5b3c69132054a8694617c5dfc030672, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/c961270271694cc990057d65c1d9504a] 2024-11-16T10:24:41,541 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02ab6a714c8748d9b56ea54299b3fa57, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731752675848 2024-11-16T10:24:41,541 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 424b7512b80a4c45a2b3beb30423a609, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731752675848 2024-11-16T10:24:41,542 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 267c0402ebb64963b28eb83f8be9ef67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731752677051 2024-11-16T10:24:41,542 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting a5b3c69132054a8694617c5dfc030672, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731752677051 2024-11-16T10:24:41,542 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4103c799c504dd18495185d25c59802, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731752679221 2024-11-16T10:24:41,542 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting c961270271694cc990057d65c1d9504a, keycount=250, bloomtype=ROW, size=47.4 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731752679221 2024-11-16T10:24:41,552 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ea6f99fb63837c16170eecaeea220bd#B#compaction#153 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:41,553 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/9ca7869a5e9045c1bda8f6631891260a is 50, key is test_row_0/B:col10/1731752679221/Put/seqid=0 2024-11-16T10:24:41,553 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:41,556 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111671105b772e554f90ae5dd6d611b8b5b5_3ea6f99fb63837c16170eecaeea220bd store=[table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:41,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742010_1186 (size=12831) 2024-11-16T10:24:41,583 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111671105b772e554f90ae5dd6d611b8b5b5_3ea6f99fb63837c16170eecaeea220bd, store=[table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:41,583 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111671105b772e554f90ae5dd6d611b8b5b5_3ea6f99fb63837c16170eecaeea220bd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:41,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-16T10:24:41,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:41,612 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 3ea6f99fb63837c16170eecaeea220bd 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T10:24:41,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742011_1187 (size=4469) 2024-11-16T10:24:41,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a9b8bd55e3894269a04be25912ce0f28_3ea6f99fb63837c16170eecaeea220bd is 50, key is test_row_0/A:col10/1731752679902/Put/seqid=0 2024-11-16T10:24:41,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742012_1188 (size=12454) 2024-11-16T10:24:41,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:41,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. as already flushing 2024-11-16T10:24:41,909 INFO [Thread-599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-599. 2024-11-16T10:24:41,909 DEBUG [Thread-599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:24:41,909 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:41,910 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:41,910 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:24:41,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T10:24:42,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:42,092 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ea6f99fb63837c16170eecaeea220bd#A#compaction#154 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:42,093 INFO [Thread-603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-603. 2024-11-16T10:24:42,093 DEBUG [Thread-603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:24:42,093 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/9ca7869a5e9045c1bda8f6631891260a as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/9ca7869a5e9045c1bda8f6631891260a 2024-11-16T10:24:42,093 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:42,093 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/c3988f7441254c3b843cafd19dfcff44 is 175, key is test_row_0/A:col10/1731752679221/Put/seqid=0 2024-11-16T10:24:42,093 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:42,093 INFO [Thread-601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-601. 2024-11-16T10:24:42,093 DEBUG [Thread-601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:24:42,093 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:42,094 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:24:42,094 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:42,094 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:24:42,098 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a9b8bd55e3894269a04be25912ce0f28_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a9b8bd55e3894269a04be25912ce0f28_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:42,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/9de0ef889dd8496b8faed7d949835749, store: [table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:42,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/9de0ef889dd8496b8faed7d949835749 is 175, key is test_row_0/A:col10/1731752679902/Put/seqid=0 2024-11-16T10:24:42,100 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ea6f99fb63837c16170eecaeea220bd/B of 3ea6f99fb63837c16170eecaeea220bd into 9ca7869a5e9045c1bda8f6631891260a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:42,100 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ea6f99fb63837c16170eecaeea220bd: 2024-11-16T10:24:42,100 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., storeName=3ea6f99fb63837c16170eecaeea220bd/B, priority=13, startTime=1731752681539; duration=0sec 2024-11-16T10:24:42,100 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:24:42,100 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ea6f99fb63837c16170eecaeea220bd:B 2024-11-16T10:24:42,100 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:24:42,101 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37031 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:24:42,102 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): 3ea6f99fb63837c16170eecaeea220bd/C is initiating minor compaction (all files) 2024-11-16T10:24:42,102 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ea6f99fb63837c16170eecaeea220bd/C in TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:42,102 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/3f788fceaebe463abc7d7816e603b44e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/7dcf13ac5094483b9ea2b4a39e4de552, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/7e0551885a914b998f6c654a43af128a] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp, totalSize=36.2 K 2024-11-16T10:24:42,102 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f788fceaebe463abc7d7816e603b44e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731752675848 2024-11-16T10:24:42,102 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7dcf13ac5094483b9ea2b4a39e4de552, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731752677051 2024-11-16T10:24:42,103 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e0551885a914b998f6c654a43af128a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731752679221 2024-11-16T10:24:42,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742013_1189 (size=31785) 2024-11-16T10:24:42,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742014_1190 (size=31255) 2024-11-16T10:24:42,112 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ea6f99fb63837c16170eecaeea220bd#C#compaction#156 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:42,113 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/d63c989734ac4e93b7ae44398c958e6d is 50, key is test_row_0/C:col10/1731752679221/Put/seqid=0 2024-11-16T10:24:42,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742015_1191 (size=12831) 2024-11-16T10:24:42,352 INFO [Thread-605 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-605. 2024-11-16T10:24:42,352 DEBUG [Thread-605 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:24:42,352 DEBUG [Thread-605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:42,352 DEBUG [Thread-605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:42,352 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:24:42,367 INFO [Thread-597 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-597. 2024-11-16T10:24:42,367 DEBUG [Thread-597 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:24:42,368 DEBUG [Thread-597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:42,368 DEBUG [Thread-597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:42,368 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:24:42,507 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=283, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/9de0ef889dd8496b8faed7d949835749 2024-11-16T10:24:42,511 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/c3988f7441254c3b843cafd19dfcff44 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/c3988f7441254c3b843cafd19dfcff44 2024-11-16T10:24:42,516 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ea6f99fb63837c16170eecaeea220bd/A of 3ea6f99fb63837c16170eecaeea220bd into c3988f7441254c3b843cafd19dfcff44(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:42,516 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ea6f99fb63837c16170eecaeea220bd: 2024-11-16T10:24:42,516 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., storeName=3ea6f99fb63837c16170eecaeea220bd/A, priority=13, startTime=1731752681539; duration=0sec 2024-11-16T10:24:42,516 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:42,516 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ea6f99fb63837c16170eecaeea220bd:A 2024-11-16T10:24:42,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/3c8952a7756347f09fe684020e4e5599 is 50, key is test_row_0/B:col10/1731752679902/Put/seqid=0 2024-11-16T10:24:42,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742016_1192 (size=12301) 2024-11-16T10:24:42,523 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/d63c989734ac4e93b7ae44398c958e6d as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/d63c989734ac4e93b7ae44398c958e6d 2024-11-16T10:24:42,528 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ea6f99fb63837c16170eecaeea220bd/C of 3ea6f99fb63837c16170eecaeea220bd into d63c989734ac4e93b7ae44398c958e6d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:42,528 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ea6f99fb63837c16170eecaeea220bd: 2024-11-16T10:24:42,528 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., storeName=3ea6f99fb63837c16170eecaeea220bd/C, priority=13, startTime=1731752681540; duration=0sec 2024-11-16T10:24:42,528 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:42,528 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ea6f99fb63837c16170eecaeea220bd:C 2024-11-16T10:24:42,923 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/3c8952a7756347f09fe684020e4e5599 2024-11-16T10:24:42,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/10abce98a2484dd7ae3d5bcbd8248c3c is 50, key is test_row_0/C:col10/1731752679902/Put/seqid=0 2024-11-16T10:24:42,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742017_1193 (size=12301) 2024-11-16T10:24:43,336 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/10abce98a2484dd7ae3d5bcbd8248c3c 2024-11-16T10:24:43,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/9de0ef889dd8496b8faed7d949835749 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/9de0ef889dd8496b8faed7d949835749 2024-11-16T10:24:43,346 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/9de0ef889dd8496b8faed7d949835749, entries=150, sequenceid=283, filesize=30.5 K 2024-11-16T10:24:43,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/3c8952a7756347f09fe684020e4e5599 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/3c8952a7756347f09fe684020e4e5599 2024-11-16T10:24:43,352 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/3c8952a7756347f09fe684020e4e5599, entries=150, sequenceid=283, filesize=12.0 K 2024-11-16T10:24:43,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/10abce98a2484dd7ae3d5bcbd8248c3c as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/10abce98a2484dd7ae3d5bcbd8248c3c 2024-11-16T10:24:43,358 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/10abce98a2484dd7ae3d5bcbd8248c3c, entries=150, sequenceid=283, filesize=12.0 K 2024-11-16T10:24:43,359 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=33.54 KB/34350 for 3ea6f99fb63837c16170eecaeea220bd in 1748ms, sequenceid=283, compaction requested=false 2024-11-16T10:24:43,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 3ea6f99fb63837c16170eecaeea220bd: 2024-11-16T10:24:43,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:43,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-16T10:24:43,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-16T10:24:43,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-16T10:24:43,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6000 sec 2024-11-16T10:24:43,364 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 3.6060 sec 2024-11-16T10:24:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T10:24:43,937 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:24:43,937 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T10:24:43,937 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-16T10:24:43,937 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-16T10:24:43,937 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-16T10:24:43,937 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-16T10:24:43,937 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-16T10:24:43,937 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T10:24:43,937 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3895 2024-11-16T10:24:43,937 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3976 2024-11-16T10:24:43,937 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T10:24:43,937 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1655 2024-11-16T10:24:43,937 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4965 rows 2024-11-16T10:24:43,937 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1671 2024-11-16T10:24:43,937 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5009 rows 2024-11-16T10:24:43,937 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T10:24:43,938 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T10:24:43,938 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:43,938 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:43,938 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:24:43,939 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T10:24:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T10:24:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T10:24:43,945 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752683945"}]},"ts":"1731752683945"} 2024-11-16T10:24:43,947 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T10:24:43,947 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T10:24:43,948 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T10:24:43,950 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ea6f99fb63837c16170eecaeea220bd, UNASSIGN}] 2024-11-16T10:24:43,951 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ea6f99fb63837c16170eecaeea220bd, UNASSIGN 2024-11-16T10:24:43,951 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=3ea6f99fb63837c16170eecaeea220bd, regionState=CLOSING, regionLocation=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:43,953 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ea6f99fb63837c16170eecaeea220bd, UNASSIGN because future has completed 2024-11-16T10:24:43,953 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T10:24:43,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506}] 2024-11-16T10:24:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T10:24:44,106 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:44,106 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T10:24:44,106 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing 3ea6f99fb63837c16170eecaeea220bd, disabling compactions & flushes 2024-11-16T10:24:44,106 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:44,106 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:44,106 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. after waiting 0 ms 2024-11-16T10:24:44,106 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:44,106 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing 3ea6f99fb63837c16170eecaeea220bd 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-16T10:24:44,113 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163f3d241c47394c12897851c55eef736b_3ea6f99fb63837c16170eecaeea220bd is 50, key is test_row_0/A:col10/1731752682091/Put/seqid=0 2024-11-16T10:24:44,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742018_1194 (size=9914) 2024-11-16T10:24:44,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T10:24:44,517 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:44,523 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163f3d241c47394c12897851c55eef736b_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163f3d241c47394c12897851c55eef736b_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:44,524 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/f78d66f7fb194b7da42915ca9a414b8d, store: [table=TestAcidGuarantees family=A region=3ea6f99fb63837c16170eecaeea220bd] 2024-11-16T10:24:44,525 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/f78d66f7fb194b7da42915ca9a414b8d is 175, key is test_row_0/A:col10/1731752682091/Put/seqid=0 2024-11-16T10:24:44,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742019_1195 (size=22561) 2024-11-16T10:24:44,529 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/f78d66f7fb194b7da42915ca9a414b8d 2024-11-16T10:24:44,536 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/81d2256dbfa64b0ca687a68cece9a99f is 50, key is test_row_0/B:col10/1731752682091/Put/seqid=0 2024-11-16T10:24:44,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742020_1196 (size=9857) 2024-11-16T10:24:44,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T10:24:44,941 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/81d2256dbfa64b0ca687a68cece9a99f 2024-11-16T10:24:44,950 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/cd152f3de02d47648406998ac2741e1c is 50, key is test_row_0/C:col10/1731752682091/Put/seqid=0 2024-11-16T10:24:44,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742021_1197 (size=9857) 2024-11-16T10:24:45,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T10:24:45,096 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T10:24:45,355 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/cd152f3de02d47648406998ac2741e1c 2024-11-16T10:24:45,360 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/A/f78d66f7fb194b7da42915ca9a414b8d as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/f78d66f7fb194b7da42915ca9a414b8d 2024-11-16T10:24:45,365 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/f78d66f7fb194b7da42915ca9a414b8d, entries=100, sequenceid=294, filesize=22.0 K 2024-11-16T10:24:45,366 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/B/81d2256dbfa64b0ca687a68cece9a99f as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/81d2256dbfa64b0ca687a68cece9a99f 2024-11-16T10:24:45,371 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/81d2256dbfa64b0ca687a68cece9a99f, entries=100, sequenceid=294, filesize=9.6 K 2024-11-16T10:24:45,372 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/.tmp/C/cd152f3de02d47648406998ac2741e1c as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/cd152f3de02d47648406998ac2741e1c 2024-11-16T10:24:45,378 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/cd152f3de02d47648406998ac2741e1c, entries=100, sequenceid=294, filesize=9.6 K 2024-11-16T10:24:45,379 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 3ea6f99fb63837c16170eecaeea220bd in 1273ms, sequenceid=294, compaction requested=true 2024-11-16T10:24:45,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/6f952ab716944e8bbd32522a3688f68c, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/0b86f0b6f0d243acbe494edb124bef52, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/b6722dfd23b9402e93f3dcdef234aa2d, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/c4e139c1b8e0402a83a8d266ae3ad5bd, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/a0e6ef3b8a794453bda3d31357cb8634, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/48561b82da054794bdac6147693fcb08, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/93d63deeb84c4a1e8891776713969219, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/e91918b706da41ab86b9627954463735, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/812f1f3e77bb444d8cf4b55a1488b5cb, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/ab8fb9106e6747b0bab94dc31413db10, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/44548865e7c84cda984a44f2c379696c, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/31cb37d772b04a629bcdd99d3f9f6ca9, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/cbe4334d0fd249f6af2a3c5169330893, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/211af02ac7844702921ba85cc989faaf, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/424b7512b80a4c45a2b3beb30423a609, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/a5b3c69132054a8694617c5dfc030672, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/c961270271694cc990057d65c1d9504a] to archive 2024-11-16T10:24:45,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:24:45,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/6f952ab716944e8bbd32522a3688f68c to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/6f952ab716944e8bbd32522a3688f68c 2024-11-16T10:24:45,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/0b86f0b6f0d243acbe494edb124bef52 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/0b86f0b6f0d243acbe494edb124bef52 2024-11-16T10:24:45,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/b6722dfd23b9402e93f3dcdef234aa2d to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/b6722dfd23b9402e93f3dcdef234aa2d 2024-11-16T10:24:45,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/c4e139c1b8e0402a83a8d266ae3ad5bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/c4e139c1b8e0402a83a8d266ae3ad5bd 2024-11-16T10:24:45,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/a0e6ef3b8a794453bda3d31357cb8634 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/a0e6ef3b8a794453bda3d31357cb8634 2024-11-16T10:24:45,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/48561b82da054794bdac6147693fcb08 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/48561b82da054794bdac6147693fcb08 2024-11-16T10:24:45,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/93d63deeb84c4a1e8891776713969219 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/93d63deeb84c4a1e8891776713969219 2024-11-16T10:24:45,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/e91918b706da41ab86b9627954463735 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/e91918b706da41ab86b9627954463735 2024-11-16T10:24:45,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/812f1f3e77bb444d8cf4b55a1488b5cb to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/812f1f3e77bb444d8cf4b55a1488b5cb 2024-11-16T10:24:45,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/ab8fb9106e6747b0bab94dc31413db10 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/ab8fb9106e6747b0bab94dc31413db10 2024-11-16T10:24:45,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/44548865e7c84cda984a44f2c379696c to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/44548865e7c84cda984a44f2c379696c 2024-11-16T10:24:45,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/31cb37d772b04a629bcdd99d3f9f6ca9 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/31cb37d772b04a629bcdd99d3f9f6ca9 2024-11-16T10:24:45,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/cbe4334d0fd249f6af2a3c5169330893 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/cbe4334d0fd249f6af2a3c5169330893 2024-11-16T10:24:45,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/211af02ac7844702921ba85cc989faaf to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/211af02ac7844702921ba85cc989faaf 2024-11-16T10:24:45,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/424b7512b80a4c45a2b3beb30423a609 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/424b7512b80a4c45a2b3beb30423a609 2024-11-16T10:24:45,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/a5b3c69132054a8694617c5dfc030672 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/a5b3c69132054a8694617c5dfc030672 2024-11-16T10:24:45,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/c961270271694cc990057d65c1d9504a to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/c961270271694cc990057d65c1d9504a 2024-11-16T10:24:45,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/93cf5096393c4bff8905931e9bfea196, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/2ae8618396c74a35ae80701ca0ada023, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/9fea73f1f6f3482992a11958aaf14064, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/ebac1cd030614e2788b2da0ae73a7167, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/826a2cbeb85a4fa78fbdf2edc1e258cc, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/dd6f25ef5f5f45788c6e93f2d0e7fbf5, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/fdca30d912dd4c55bc401451277ddbb0, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/032ba778854c45019f051c62963362ad, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/e591290d60444334aee6641ec143917d, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/8291850e3f524b818c1d21d06e97af24, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/cb5a722a09cb45a2a9691baf50fb46a4, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/f8607285b10b43379ad522feb16feb95, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/060ea2614e6b457a87d10ce488894023, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/02ab6a714c8748d9b56ea54299b3fa57, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/f821f952ba9146fd8d9057419d55eb26, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/267c0402ebb64963b28eb83f8be9ef67, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/b4103c799c504dd18495185d25c59802] to archive 2024-11-16T10:24:45,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:24:45,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/93cf5096393c4bff8905931e9bfea196 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/93cf5096393c4bff8905931e9bfea196 2024-11-16T10:24:45,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/2ae8618396c74a35ae80701ca0ada023 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/2ae8618396c74a35ae80701ca0ada023 2024-11-16T10:24:45,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/9fea73f1f6f3482992a11958aaf14064 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/9fea73f1f6f3482992a11958aaf14064 2024-11-16T10:24:45,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/ebac1cd030614e2788b2da0ae73a7167 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/ebac1cd030614e2788b2da0ae73a7167 2024-11-16T10:24:45,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/826a2cbeb85a4fa78fbdf2edc1e258cc to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/826a2cbeb85a4fa78fbdf2edc1e258cc 2024-11-16T10:24:45,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/dd6f25ef5f5f45788c6e93f2d0e7fbf5 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/dd6f25ef5f5f45788c6e93f2d0e7fbf5 2024-11-16T10:24:45,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/fdca30d912dd4c55bc401451277ddbb0 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/fdca30d912dd4c55bc401451277ddbb0 2024-11-16T10:24:45,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/032ba778854c45019f051c62963362ad to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/032ba778854c45019f051c62963362ad 2024-11-16T10:24:45,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/e591290d60444334aee6641ec143917d to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/e591290d60444334aee6641ec143917d 2024-11-16T10:24:45,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/8291850e3f524b818c1d21d06e97af24 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/8291850e3f524b818c1d21d06e97af24 2024-11-16T10:24:45,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/cb5a722a09cb45a2a9691baf50fb46a4 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/cb5a722a09cb45a2a9691baf50fb46a4 2024-11-16T10:24:45,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/f8607285b10b43379ad522feb16feb95 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/f8607285b10b43379ad522feb16feb95 2024-11-16T10:24:45,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/060ea2614e6b457a87d10ce488894023 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/060ea2614e6b457a87d10ce488894023 2024-11-16T10:24:45,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/02ab6a714c8748d9b56ea54299b3fa57 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/02ab6a714c8748d9b56ea54299b3fa57 2024-11-16T10:24:45,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/f821f952ba9146fd8d9057419d55eb26 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/f821f952ba9146fd8d9057419d55eb26 2024-11-16T10:24:45,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/267c0402ebb64963b28eb83f8be9ef67 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/267c0402ebb64963b28eb83f8be9ef67 2024-11-16T10:24:45,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/b4103c799c504dd18495185d25c59802 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/b4103c799c504dd18495185d25c59802 2024-11-16T10:24:45,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/dce857a9156c42b198dd8000a4037503, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/5e1c5e15cfe142358bfe4bf487f7142b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/36bae6a323e34df79066d7016c436c22, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/7ee63e453b99417bb9cd6e1a2f7b5a01, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/eb324486126c49bd8333bcaf45b90621, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/b5a564824e2b49aa8aa80572d6d623cb, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/56272c26a0b141cc9928013718507b1e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/f3f5a923f7e24768ac89df17ec3dd173, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/048e510581504ba18eef71e767f03f0c, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/4a45d7d0a84149b097b8a2b653a6d82b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/7ea8bb3cfe0b4b3baaf0b4d24787bcd8, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/14446f32cb454cce8bf96ffdfe7da3e1, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/af6b06f2c988471a9bc53156253e95eb, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/3f788fceaebe463abc7d7816e603b44e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/b9e51a94acb640d794febb9c587bc6ae, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/7dcf13ac5094483b9ea2b4a39e4de552, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/7e0551885a914b998f6c654a43af128a] to archive 2024-11-16T10:24:45,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:24:45,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/dce857a9156c42b198dd8000a4037503 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/dce857a9156c42b198dd8000a4037503 2024-11-16T10:24:45,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/5e1c5e15cfe142358bfe4bf487f7142b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/5e1c5e15cfe142358bfe4bf487f7142b 2024-11-16T10:24:45,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/36bae6a323e34df79066d7016c436c22 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/36bae6a323e34df79066d7016c436c22 2024-11-16T10:24:45,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/7ee63e453b99417bb9cd6e1a2f7b5a01 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/7ee63e453b99417bb9cd6e1a2f7b5a01 2024-11-16T10:24:45,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/eb324486126c49bd8333bcaf45b90621 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/eb324486126c49bd8333bcaf45b90621 2024-11-16T10:24:45,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/b5a564824e2b49aa8aa80572d6d623cb to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/b5a564824e2b49aa8aa80572d6d623cb 2024-11-16T10:24:45,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/56272c26a0b141cc9928013718507b1e to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/56272c26a0b141cc9928013718507b1e 2024-11-16T10:24:45,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/f3f5a923f7e24768ac89df17ec3dd173 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/f3f5a923f7e24768ac89df17ec3dd173 2024-11-16T10:24:45,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/048e510581504ba18eef71e767f03f0c to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/048e510581504ba18eef71e767f03f0c 2024-11-16T10:24:45,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/4a45d7d0a84149b097b8a2b653a6d82b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/4a45d7d0a84149b097b8a2b653a6d82b 2024-11-16T10:24:45,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/7ea8bb3cfe0b4b3baaf0b4d24787bcd8 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/7ea8bb3cfe0b4b3baaf0b4d24787bcd8 2024-11-16T10:24:45,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/14446f32cb454cce8bf96ffdfe7da3e1 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/14446f32cb454cce8bf96ffdfe7da3e1 2024-11-16T10:24:45,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/af6b06f2c988471a9bc53156253e95eb to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/af6b06f2c988471a9bc53156253e95eb 2024-11-16T10:24:45,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/3f788fceaebe463abc7d7816e603b44e to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/3f788fceaebe463abc7d7816e603b44e 2024-11-16T10:24:45,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/b9e51a94acb640d794febb9c587bc6ae to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/b9e51a94acb640d794febb9c587bc6ae 2024-11-16T10:24:45,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/7dcf13ac5094483b9ea2b4a39e4de552 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/7dcf13ac5094483b9ea2b4a39e4de552 2024-11-16T10:24:45,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/7e0551885a914b998f6c654a43af128a to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/7e0551885a914b998f6c654a43af128a 2024-11-16T10:24:45,466 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/recovered.edits/297.seqid, newMaxSeqId=297, maxSeqId=4 2024-11-16T10:24:45,467 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. 2024-11-16T10:24:45,467 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for 3ea6f99fb63837c16170eecaeea220bd: Waiting for close lock at 1731752684106Running coprocessor pre-close hooks at 1731752684106Disabling compacts and flushes for region at 1731752684106Disabling writes for close at 1731752684106Obtaining lock to block concurrent updates at 1731752684106Preparing flush snapshotting stores in 3ea6f99fb63837c16170eecaeea220bd at 1731752684106Finished memstore snapshotting TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731752684106Flushing stores of TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd. at 1731752684107 (+1 ms)Flushing 3ea6f99fb63837c16170eecaeea220bd/A: creating writer at 1731752684107Flushing mob file 3ea6f99fb63837c16170eecaeea220bd/A: appending metadata at 1731752684113 (+6 ms)Flushing mob file 3ea6f99fb63837c16170eecaeea220bd/A: closing flushed file at 1731752684113Flushing 3ea6f99fb63837c16170eecaeea220bd/A: appending metadata at 1731752684524 (+411 ms)Flushing 3ea6f99fb63837c16170eecaeea220bd/A: closing flushed file at 1731752684524Flushing 3ea6f99fb63837c16170eecaeea220bd/B: creating writer at 1731752684533 (+9 ms)Flushing 3ea6f99fb63837c16170eecaeea220bd/B: appending metadata at 1731752684536 (+3 ms)Flushing 3ea6f99fb63837c16170eecaeea220bd/B: closing flushed file at 1731752684536Flushing 3ea6f99fb63837c16170eecaeea220bd/C: creating writer at 1731752684946 (+410 ms)Flushing 3ea6f99fb63837c16170eecaeea220bd/C: appending metadata at 1731752684949 (+3 ms)Flushing 3ea6f99fb63837c16170eecaeea220bd/C: closing flushed file at 1731752684949Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fab1df9: reopening flushed file at 1731752685359 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26c41cf5: reopening flushed file at 1731752685365 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29f9dc00: reopening flushed file at 1731752685371 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 3ea6f99fb63837c16170eecaeea220bd in 1273ms, sequenceid=294, compaction requested=true at 1731752685379 (+8 ms)Writing region close event to WAL at 1731752685463 (+84 ms)Running coprocessor post-close hooks at 1731752685467 (+4 ms)Closed at 1731752685467 2024-11-16T10:24:45,469 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed 3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:45,470 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=3ea6f99fb63837c16170eecaeea220bd, regionState=CLOSED 2024-11-16T10:24:45,471 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 because future has completed 2024-11-16T10:24:45,475 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-16T10:24:45,475 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure 3ea6f99fb63837c16170eecaeea220bd, server=b16ff38458c1,35873,1731752627506 in 1.5190 sec 2024-11-16T10:24:45,476 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-16T10:24:45,476 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ea6f99fb63837c16170eecaeea220bd, UNASSIGN in 1.5250 sec 2024-11-16T10:24:45,479 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-16T10:24:45,479 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5290 sec 2024-11-16T10:24:45,480 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752685480"}]},"ts":"1731752685480"} 2024-11-16T10:24:45,481 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T10:24:45,481 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T10:24:45,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5420 sec 2024-11-16T10:24:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T10:24:46,087 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:24:46,087 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T10:24:46,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:24:46,089 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:24:46,090 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:24:46,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-16T10:24:46,092 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:46,094 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A, FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B, FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C, FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/recovered.edits] 2024-11-16T10:24:46,097 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/9de0ef889dd8496b8faed7d949835749 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/9de0ef889dd8496b8faed7d949835749 2024-11-16T10:24:46,099 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/c3988f7441254c3b843cafd19dfcff44 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/c3988f7441254c3b843cafd19dfcff44 2024-11-16T10:24:46,100 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/f78d66f7fb194b7da42915ca9a414b8d to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/A/f78d66f7fb194b7da42915ca9a414b8d 2024-11-16T10:24:46,102 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/3c8952a7756347f09fe684020e4e5599 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/3c8952a7756347f09fe684020e4e5599 2024-11-16T10:24:46,103 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/81d2256dbfa64b0ca687a68cece9a99f to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/81d2256dbfa64b0ca687a68cece9a99f 2024-11-16T10:24:46,105 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/9ca7869a5e9045c1bda8f6631891260a to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/B/9ca7869a5e9045c1bda8f6631891260a 2024-11-16T10:24:46,107 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/10abce98a2484dd7ae3d5bcbd8248c3c to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/10abce98a2484dd7ae3d5bcbd8248c3c 2024-11-16T10:24:46,108 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/cd152f3de02d47648406998ac2741e1c to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/cd152f3de02d47648406998ac2741e1c 2024-11-16T10:24:46,109 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/d63c989734ac4e93b7ae44398c958e6d to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/C/d63c989734ac4e93b7ae44398c958e6d 2024-11-16T10:24:46,111 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/recovered.edits/297.seqid to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd/recovered.edits/297.seqid 2024-11-16T10:24:46,112 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:46,112 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T10:24:46,112 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T10:24:46,113 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-16T10:24:46,116 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111604cf8b2dc5f24b70b50986f5d6bf5fdd_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111604cf8b2dc5f24b70b50986f5d6bf5fdd_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:46,117 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161c767dd47f4e4b6eac9aaf534b6ebfa7_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161c767dd47f4e4b6eac9aaf534b6ebfa7_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:46,118 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163622bf5ea8684e939038c12090e5c62b_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163622bf5ea8684e939038c12090e5c62b_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:46,119 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111636b48292d49a4b3292867082252dc1cf_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111636b48292d49a4b3292867082252dc1cf_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:46,120 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163f3d241c47394c12897851c55eef736b_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163f3d241c47394c12897851c55eef736b_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:46,121 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116408e7cd2260d4fbf80eaaeb1699dc179_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116408e7cd2260d4fbf80eaaeb1699dc179_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:46,122 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165d8b80ca388b4f42a8c5bcf189769b5e_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165d8b80ca388b4f42a8c5bcf189769b5e_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:46,123 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111660313e998a574a7ca1820ac16875ebe3_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111660313e998a574a7ca1820ac16875ebe3_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:46,124 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a9b8bd55e3894269a04be25912ce0f28_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a9b8bd55e3894269a04be25912ce0f28_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:46,125 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b6899a112f724f62bd9b47ff771dcd29_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b6899a112f724f62bd9b47ff771dcd29_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:46,126 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b75d881d6d5e480d8f5ebc24f0058280_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b75d881d6d5e480d8f5ebc24f0058280_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:46,127 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c4811a352adc40b3881884f03e73266d_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c4811a352adc40b3881884f03e73266d_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:46,128 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c4e70fa6468a4207b5c59ad363ce0d5a_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c4e70fa6468a4207b5c59ad363ce0d5a_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:46,128 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c8cda2f6b76d4f5896d784d577552d01_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c8cda2f6b76d4f5896d784d577552d01_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:46,129 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116eafacd4c026645dfb1ad97fd09cd4ca9_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116eafacd4c026645dfb1ad97fd09cd4ca9_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:46,130 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fa08956d5e7848e0ab50b4f0ad868611_3ea6f99fb63837c16170eecaeea220bd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fa08956d5e7848e0ab50b4f0ad868611_3ea6f99fb63837c16170eecaeea220bd 2024-11-16T10:24:46,131 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T10:24:46,133 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:24:46,134 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T10:24:46,136 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T10:24:46,137 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:24:46,137 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T10:24:46,138 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731752686137"}]},"ts":"9223372036854775807"} 2024-11-16T10:24:46,140 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T10:24:46,140 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3ea6f99fb63837c16170eecaeea220bd, NAME => 'TestAcidGuarantees,,1731752657328.3ea6f99fb63837c16170eecaeea220bd.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T10:24:46,140 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T10:24:46,140 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731752686140"}]},"ts":"9223372036854775807"} 2024-11-16T10:24:46,142 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T10:24:46,143 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:24:46,144 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 56 msec 2024-11-16T10:24:46,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-16T10:24:46,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T10:24:46,197 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:24:46,205 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=241 (was 239) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107388350_22 at /127.0.0.1:41688 [Waiting for operation #605] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1713545760_22 at /127.0.0.1:35544 [Waiting for operation #633] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/cluster_7ec35abe-b1d1-d9da-1787-50b9d114d89d/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107388350_22 at /127.0.0.1:59594 [Waiting for operation #919] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/cluster_7ec35abe-b1d1-d9da-1787-50b9d114d89d/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=497 (was 552), ProcessCount=11 (was 11), AvailableMemoryMB=4078 (was 4359) 2024-11-16T10:24:46,214 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=241, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=497, ProcessCount=11, AvailableMemoryMB=4077 2024-11-16T10:24:46,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T10:24:46,216 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T10:24:46,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T10:24:46,218 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T10:24:46,218 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:46,218 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-16T10:24:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-16T10:24:46,219 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T10:24:46,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742022_1198 (size=959) 2024-11-16T10:24:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-16T10:24:46,495 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T10:24:46,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-16T10:24:46,652 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7 2024-11-16T10:24:46,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742023_1199 (size=53) 2024-11-16T10:24:46,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-16T10:24:47,064 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:24:47,065 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d41ecfe9dd6b68876325d42387a8c316, disabling compactions & flushes 2024-11-16T10:24:47,065 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:47,065 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:47,065 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. after waiting 0 ms 2024-11-16T10:24:47,065 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:47,065 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:47,065 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d41ecfe9dd6b68876325d42387a8c316: Waiting for close lock at 1731752687065Disabling compacts and flushes for region at 1731752687065Disabling writes for close at 1731752687065Writing region close event to WAL at 1731752687065Closed at 1731752687065 2024-11-16T10:24:47,068 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T10:24:47,069 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731752687068"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731752687068"}]},"ts":"1731752687068"} 2024-11-16T10:24:47,073 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T10:24:47,075 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T10:24:47,075 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752687075"}]},"ts":"1731752687075"} 2024-11-16T10:24:47,076 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T10:24:47,077 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d41ecfe9dd6b68876325d42387a8c316, ASSIGN}] 2024-11-16T10:24:47,078 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d41ecfe9dd6b68876325d42387a8c316, ASSIGN 2024-11-16T10:24:47,079 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d41ecfe9dd6b68876325d42387a8c316, ASSIGN; state=OFFLINE, location=b16ff38458c1,35873,1731752627506; forceNewPlan=false, retain=false 2024-11-16T10:24:47,229 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=d41ecfe9dd6b68876325d42387a8c316, regionState=OPENING, regionLocation=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:47,231 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d41ecfe9dd6b68876325d42387a8c316, ASSIGN because future has completed 2024-11-16T10:24:47,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506}] 2024-11-16T10:24:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-16T10:24:47,391 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:47,391 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} 2024-11-16T10:24:47,392 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:24:47,392 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:24:47,392 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:24:47,392 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:24:47,396 INFO [StoreOpener-d41ecfe9dd6b68876325d42387a8c316-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:24:47,398 INFO [StoreOpener-d41ecfe9dd6b68876325d42387a8c316-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d41ecfe9dd6b68876325d42387a8c316 columnFamilyName A 2024-11-16T10:24:47,399 DEBUG [StoreOpener-d41ecfe9dd6b68876325d42387a8c316-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:47,400 INFO [StoreOpener-d41ecfe9dd6b68876325d42387a8c316-1 {}] regionserver.HStore(327): Store=d41ecfe9dd6b68876325d42387a8c316/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:24:47,400 INFO [StoreOpener-d41ecfe9dd6b68876325d42387a8c316-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:24:47,401 INFO [StoreOpener-d41ecfe9dd6b68876325d42387a8c316-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d41ecfe9dd6b68876325d42387a8c316 columnFamilyName B 2024-11-16T10:24:47,401 DEBUG [StoreOpener-d41ecfe9dd6b68876325d42387a8c316-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:47,402 INFO [StoreOpener-d41ecfe9dd6b68876325d42387a8c316-1 {}] regionserver.HStore(327): Store=d41ecfe9dd6b68876325d42387a8c316/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:24:47,402 INFO [StoreOpener-d41ecfe9dd6b68876325d42387a8c316-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:24:47,404 INFO [StoreOpener-d41ecfe9dd6b68876325d42387a8c316-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d41ecfe9dd6b68876325d42387a8c316 columnFamilyName C 2024-11-16T10:24:47,404 DEBUG [StoreOpener-d41ecfe9dd6b68876325d42387a8c316-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:24:47,404 INFO [StoreOpener-d41ecfe9dd6b68876325d42387a8c316-1 {}] regionserver.HStore(327): Store=d41ecfe9dd6b68876325d42387a8c316/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:24:47,404 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:24:47,405 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:24:47,406 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:24:47,407 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:24:47,407 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:24:47,408 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T10:24:47,409 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:24:47,412 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T10:24:47,412 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened d41ecfe9dd6b68876325d42387a8c316; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67444059, jitterRate=0.004994794726371765}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T10:24:47,412 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:24:47,413 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for d41ecfe9dd6b68876325d42387a8c316: Running coprocessor pre-open hook at 1731752687392Writing region info on filesystem at 1731752687392Initializing all the Stores at 1731752687394 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752687395 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752687395Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752687395Cleaning up temporary data from old regions at 1731752687407 (+12 ms)Running coprocessor post-open hooks at 1731752687412 (+5 ms)Region opened successfully at 1731752687413 (+1 ms) 2024-11-16T10:24:47,414 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., pid=63, masterSystemTime=1731752687384 2024-11-16T10:24:47,416 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:47,416 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:47,416 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=d41ecfe9dd6b68876325d42387a8c316, regionState=OPEN, openSeqNum=2, regionLocation=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:47,418 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 because future has completed 2024-11-16T10:24:47,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-16T10:24:47,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 in 187 msec 2024-11-16T10:24:47,424 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-16T10:24:47,424 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d41ecfe9dd6b68876325d42387a8c316, ASSIGN in 344 msec 2024-11-16T10:24:47,425 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T10:24:47,425 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752687425"}]},"ts":"1731752687425"} 2024-11-16T10:24:47,427 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T10:24:47,428 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T10:24:47,430 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2120 sec 2024-11-16T10:24:48,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-16T10:24:48,366 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:24:48,368 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe11fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,368 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:24:48,368 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:24:48,369 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:24:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:24:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:24:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6be5c7b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:24:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:24:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:48,372 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:24:48,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@944096e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,373 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:24:48,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:24:48,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:24:48,376 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:24:48,377 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T10:24:48,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is b16ff38458c1,38809,1731752626047 2024-11-16T10:24:48,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4d6eeb66 2024-11-16T10:24:48,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T10:24:48,380 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T10:24:48,382 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,383 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:24:48,383 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:24:48,384 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:24:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:24:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:24:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@499ca39e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:24:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:24:48,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:48,385 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:24:48,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34e35d16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,387 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,387 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:24:48,387 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:24:48,388 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:24:48,388 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:24:48,388 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:24:48,389 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56c96093, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,389 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:24:48,389 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:24:48,389 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:48,390 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:24:48,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59683a24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,391 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,391 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:24:48,392 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:24:48,392 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:24:48,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:24:48,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:24:48,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77b08240, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:24:48,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:24:48,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:48,394 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:24:48,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46c15407, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,396 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,396 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:24:48,396 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:24:48,397 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:24:48,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:24:48,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:24:48,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28eecc9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:24:48,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:24:48,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:48,399 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:24:48,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@507f4493, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,400 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,400 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:24:48,400 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:24:48,401 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:24:48,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:24:48,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:24:48,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@afe2b81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:24:48,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:24:48,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:48,403 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46020, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:24:48,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23006c15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,404 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,404 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:24:48,404 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:24:48,405 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:24:48,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:24:48,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:24:48,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fa4f055, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:24:48,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:24:48,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:48,407 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:24:48,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ecde523, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,408 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,408 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:24:48,408 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:24:48,408 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:24:48,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:24:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:24:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ec9903e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:24:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:24:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:48,410 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:24:48,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@206ba17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,411 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,411 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:24:48,411 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:24:48,411 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:24:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:24:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:24:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f6e9ef6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:24:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:24:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:48,413 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:24:48,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42e32581, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,414 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:24:48,414 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:24:48,415 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:24:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:24:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:24:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ad10086, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:24:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:24:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:48,416 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:24:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31738d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,417 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,417 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:24:48,417 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:24:48,418 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:24:48,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:24:48,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:24:48,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ec6c9a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:24:48,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:24:48,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:24:48,419 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46120, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:24:48,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e277b94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:24:48,420 DEBUG [Thread-964 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:24:48,420 DEBUG [Thread-957 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:24:48,420 DEBUG [Thread-966 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:24:48,420 DEBUG [Thread-970 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:24:48,420 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:24:48,421 DEBUG [Thread-953 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:24:48,421 DEBUG [Thread-972 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:24:48,421 DEBUG [Thread-955 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:24:48,421 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:24:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:24:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:24:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:24:48,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-16T10:24:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:24:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:24:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:24:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:24:48,422 DEBUG [Thread-959 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:24:48,422 DEBUG [Thread-961 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:24:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:24:48,423 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:24:48,423 DEBUG [Thread-968 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:24:48,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:24:48,423 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:24:48,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:24:48,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:24:48,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:24:48,424 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:24:48,424 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:24:48,424 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:24:48,425 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:24:48,425 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:24:48,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T10:24:48,425 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:24:48,425 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:24:48,425 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:24:48,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:24:48,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:24:48,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:24:48,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:24:48,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:24:48,426 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:24:48,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:24:48,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2] 2024-11-16T10:24:48,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2] 2024-11-16T10:24:48,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:24:48,428 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:24:48,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2] 2024-11-16T10:24:48,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2] 2024-11-16T10:24:48,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2] 2024-11-16T10:24:48,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2] 2024-11-16T10:24:48,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2] 2024-11-16T10:24:48,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2] 2024-11-16T10:24:48,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2] 2024-11-16T10:24:48,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:24:48,437 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:24:48,437 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:24:48,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d41ecfe9dd6b68876325d42387a8c316 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T10:24:48,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2] 2024-11-16T10:24:48,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/079c5f79d974494f962ca2ab87ff4e10 is 50, key is test_row_0/A:col10/1731752688435/Put/seqid=0 2024-11-16T10:24:48,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34482 deadline: 1731752748464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:48,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752748464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:48,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:48,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752748464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752748465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:48,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752748466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:48,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742024_1200 (size=12001) 2024-11-16T10:24:48,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T10:24:48,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:48,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T10:24:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752748577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:48,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:48,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:48,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:48,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752748577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:48,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:48,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34482 deadline: 1731752748577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:48,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:48,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752748577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:48,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:48,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:48,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752748577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:48,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:48,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T10:24:48,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:48,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:48,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:48,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:48,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:48,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:48,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T10:24:48,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752748787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:48,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752748787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:48,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:48,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:48,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:48,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34482 deadline: 1731752748787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:48,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:48,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752748787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:48,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:48,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752748787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:48,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:48,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:48,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:48,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:48,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T10:24:48,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:48,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:48,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:48,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:48,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:48,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:48,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/079c5f79d974494f962ca2ab87ff4e10 2024-11-16T10:24:48,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/5b68f06626db45c9bf2da72332508d3a is 50, key is test_row_0/B:col10/1731752688435/Put/seqid=0 2024-11-16T10:24:48,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742025_1201 (size=12001) 2024-11-16T10:24:48,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/5b68f06626db45c9bf2da72332508d3a 2024-11-16T10:24:48,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/eafbc0a753484a38b62bfbab850c5317 is 50, key is test_row_0/C:col10/1731752688435/Put/seqid=0 2024-11-16T10:24:48,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742026_1202 (size=12001) 2024-11-16T10:24:48,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/eafbc0a753484a38b62bfbab850c5317 2024-11-16T10:24:48,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/079c5f79d974494f962ca2ab87ff4e10 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/079c5f79d974494f962ca2ab87ff4e10 2024-11-16T10:24:49,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/079c5f79d974494f962ca2ab87ff4e10, entries=150, sequenceid=14, filesize=11.7 K 2024-11-16T10:24:49,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/5b68f06626db45c9bf2da72332508d3a as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/5b68f06626db45c9bf2da72332508d3a 2024-11-16T10:24:49,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/5b68f06626db45c9bf2da72332508d3a, entries=150, sequenceid=14, filesize=11.7 K 2024-11-16T10:24:49,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/eafbc0a753484a38b62bfbab850c5317 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/eafbc0a753484a38b62bfbab850c5317 2024-11-16T10:24:49,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T10:24:49,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:49,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:49,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:49,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:49,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:49,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:49,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/eafbc0a753484a38b62bfbab850c5317, entries=150, sequenceid=14, filesize=11.7 K 2024-11-16T10:24:49,042 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d41ecfe9dd6b68876325d42387a8c316 in 604ms, sequenceid=14, compaction requested=false 2024-11-16T10:24:49,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:24:49,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T10:24:49,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:24:49,098 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:24:49,098 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d41ecfe9dd6b68876325d42387a8c316 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T10:24:49,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:49,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752749108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:49,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:49,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:49,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34482 deadline: 1731752749109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:49,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:49,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:49,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752749110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:49,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:49,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752749109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:49,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:49,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:49,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:49,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752749113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:49,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/46753f832d53430a9e8c5621eadd8ede is 50, key is test_row_0/A:col10/1731752688464/Put/seqid=0 2024-11-16T10:24:49,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:49,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742027_1203 (size=16681) 2024-11-16T10:24:49,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T10:24:49,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:49,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:49,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:49,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:49,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:49,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:49,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:49,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752749216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:49,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:49,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:49,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34482 deadline: 1731752749226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:49,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752749227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:49,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:49,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752749227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:49,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752749227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:49,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T10:24:49,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:49,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:49,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:49,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:49,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:49,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:49,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752749426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:49,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34482 deadline: 1731752749436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:49,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:49,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752749436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:49,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:49,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752749436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:49,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:49,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:49,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752749436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:49,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:49,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T10:24:49,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:49,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:49,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:49,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:49,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:49,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:49,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/46753f832d53430a9e8c5621eadd8ede 2024-11-16T10:24:49,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/7ff10646cb4f4111afc4d186044b8e34 is 50, key is test_row_0/B:col10/1731752688464/Put/seqid=0 2024-11-16T10:24:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T10:24:49,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742028_1204 (size=12001) 2024-11-16T10:24:49,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/7ff10646cb4f4111afc4d186044b8e34 2024-11-16T10:24:49,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/9615d03983c64ead93a5add93495ae12 is 50, key is test_row_0/C:col10/1731752688464/Put/seqid=0 2024-11-16T10:24:49,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T10:24:49,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:49,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:49,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:49,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:49,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:49,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742029_1205 (size=12001) 2024-11-16T10:24:49,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:49,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752749736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:49,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:49,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:49,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34482 deadline: 1731752749747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:49,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:49,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:49,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752749747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:49,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:49,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752749747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:49,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:49,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:49,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:49,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752749747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:49,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:49,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:49,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T10:24:49,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:49,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:49,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:49,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:49,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:49,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:49,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T10:24:49,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:49,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:49,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:49,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:49,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:49,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:50,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/9615d03983c64ead93a5add93495ae12 2024-11-16T10:24:50,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/46753f832d53430a9e8c5621eadd8ede as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/46753f832d53430a9e8c5621eadd8ede 2024-11-16T10:24:50,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/46753f832d53430a9e8c5621eadd8ede, entries=250, sequenceid=40, filesize=16.3 K 2024-11-16T10:24:50,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/7ff10646cb4f4111afc4d186044b8e34 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/7ff10646cb4f4111afc4d186044b8e34 2024-11-16T10:24:50,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/7ff10646cb4f4111afc4d186044b8e34, entries=150, sequenceid=40, filesize=11.7 K 2024-11-16T10:24:50,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/9615d03983c64ead93a5add93495ae12 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/9615d03983c64ead93a5add93495ae12 2024-11-16T10:24:50,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/9615d03983c64ead93a5add93495ae12, entries=150, sequenceid=40, filesize=11.7 K 2024-11-16T10:24:50,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for d41ecfe9dd6b68876325d42387a8c316 in 1005ms, sequenceid=40, compaction requested=false 2024-11-16T10:24:50,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:24:50,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T10:24:50,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:50,118 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing d41ecfe9dd6b68876325d42387a8c316 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-16T10:24:50,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/65a8a78854f340f7b4c6d253633ff9b7 is 50, key is test_row_0/A:col10/1731752689108/Put/seqid=0 2024-11-16T10:24:50,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742030_1206 (size=12001) 2024-11-16T10:24:50,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:24:50,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:50,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:50,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752750329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:50,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:50,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:50,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34482 deadline: 1731752750340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:50,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:50,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752750341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:50,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:50,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:50,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:50,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752750342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:50,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:50,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:50,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752750347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:50,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:50,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:50,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752750447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:50,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:50,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:50,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752750466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:50,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752750466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:50,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:50,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34482 deadline: 1731752750466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:50,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:50,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:50,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752750466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:50,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:50,545 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/65a8a78854f340f7b4c6d253633ff9b7 2024-11-16T10:24:50,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T10:24:50,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/7f64b0b2c30d48d889888270a77a4730 is 50, key is test_row_0/B:col10/1731752689108/Put/seqid=0 2024-11-16T10:24:50,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742031_1207 (size=12001) 2024-11-16T10:24:50,627 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/7f64b0b2c30d48d889888270a77a4730 2024-11-16T10:24:50,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/cb0a202d233b48fd85a3911f0e19191e is 50, key is test_row_0/C:col10/1731752689108/Put/seqid=0 2024-11-16T10:24:50,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742032_1208 (size=12001) 2024-11-16T10:24:50,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:50,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752750656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:50,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:50,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34482 deadline: 1731752750676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:50,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:50,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752750676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:50,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:50,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752750677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:50,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752750686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:50,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:50,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:50,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752750966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:50,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:50,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:50,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34482 deadline: 1731752750987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:50,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:50,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752750987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:50,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:50,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:50,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:50,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752750987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:50,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:50,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:50,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752750996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:50,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:50,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:51,011 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T10:24:51,059 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/cb0a202d233b48fd85a3911f0e19191e 2024-11-16T10:24:51,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/65a8a78854f340f7b4c6d253633ff9b7 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/65a8a78854f340f7b4c6d253633ff9b7 2024-11-16T10:24:51,071 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/65a8a78854f340f7b4c6d253633ff9b7, entries=150, sequenceid=50, filesize=11.7 K 2024-11-16T10:24:51,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/7f64b0b2c30d48d889888270a77a4730 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/7f64b0b2c30d48d889888270a77a4730 2024-11-16T10:24:51,078 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/7f64b0b2c30d48d889888270a77a4730, entries=150, sequenceid=50, filesize=11.7 K 2024-11-16T10:24:51,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/cb0a202d233b48fd85a3911f0e19191e as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/cb0a202d233b48fd85a3911f0e19191e 2024-11-16T10:24:51,095 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/cb0a202d233b48fd85a3911f0e19191e, entries=150, sequenceid=50, filesize=11.7 K 2024-11-16T10:24:51,096 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for d41ecfe9dd6b68876325d42387a8c316 in 978ms, sequenceid=50, compaction requested=true 2024-11-16T10:24:51,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:24:51,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:51,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-16T10:24:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-16T10:24:51,102 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-16T10:24:51,102 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6730 sec 2024-11-16T10:24:51,105 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.6830 sec 2024-11-16T10:24:51,480 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:24:51,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d41ecfe9dd6b68876325d42387a8c316 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-16T10:24:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:24:51,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/0d83833f0cc6409aaef32788d6b008e4 is 50, key is test_row_0/A:col10/1731752691478/Put/seqid=0 2024-11-16T10:24:51,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752751499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:51,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:51,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:51,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752751501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:51,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:51,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:51,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:51,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:51,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34482 deadline: 1731752751502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:51,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752751505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:51,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752751506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:51,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:51,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:51,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:51,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742033_1209 (size=14341) 2024-11-16T10:24:51,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/0d83833f0cc6409aaef32788d6b008e4 2024-11-16T10:24:51,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/721baee8f1624cf4ac4f032c20745ebb is 50, key is test_row_0/B:col10/1731752691478/Put/seqid=0 2024-11-16T10:24:51,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742034_1210 (size=12001) 2024-11-16T10:24:51,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/721baee8f1624cf4ac4f032c20745ebb 2024-11-16T10:24:51,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/a7d3c98134794f0abd6f182178564d1c is 50, key is test_row_0/C:col10/1731752691478/Put/seqid=0 2024-11-16T10:24:51,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742035_1211 (size=12001) 2024-11-16T10:24:51,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752751616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:51,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:51,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:51,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:51,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:51,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34482 deadline: 1731752751616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752751616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:51,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:51,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:51,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752751616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:51,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:51,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:51,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:51,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:51,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752751826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:51,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:51,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:51,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:51,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34482 deadline: 1731752751826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:51,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:51,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:51,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:51,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:51,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752751826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:51,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:51,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:51,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:51,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752751826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:51,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:51,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:51,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:51,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:52,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/a7d3c98134794f0abd6f182178564d1c 2024-11-16T10:24:52,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/0d83833f0cc6409aaef32788d6b008e4 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/0d83833f0cc6409aaef32788d6b008e4 2024-11-16T10:24:52,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/0d83833f0cc6409aaef32788d6b008e4, entries=200, sequenceid=78, filesize=14.0 K 2024-11-16T10:24:52,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/721baee8f1624cf4ac4f032c20745ebb as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/721baee8f1624cf4ac4f032c20745ebb 2024-11-16T10:24:52,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/721baee8f1624cf4ac4f032c20745ebb, entries=150, sequenceid=78, filesize=11.7 K 2024-11-16T10:24:52,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/a7d3c98134794f0abd6f182178564d1c as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/a7d3c98134794f0abd6f182178564d1c 2024-11-16T10:24:52,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/a7d3c98134794f0abd6f182178564d1c, entries=150, sequenceid=78, filesize=11.7 K 2024-11-16T10:24:52,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for d41ecfe9dd6b68876325d42387a8c316 in 548ms, sequenceid=78, compaction requested=true 2024-11-16T10:24:52,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:24:52,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d41ecfe9dd6b68876325d42387a8c316:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:24:52,028 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:24:52,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:52,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d41ecfe9dd6b68876325d42387a8c316:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:24:52,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:52,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d41ecfe9dd6b68876325d42387a8c316:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:24:52,028 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:24:52,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:24:52,029 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55024 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:24:52,029 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:24:52,029 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): d41ecfe9dd6b68876325d42387a8c316/A is initiating minor compaction (all files) 2024-11-16T10:24:52,029 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): d41ecfe9dd6b68876325d42387a8c316/B is initiating minor compaction (all files) 2024-11-16T10:24:52,030 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d41ecfe9dd6b68876325d42387a8c316/A in TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:52,030 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d41ecfe9dd6b68876325d42387a8c316/B in TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:52,030 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/079c5f79d974494f962ca2ab87ff4e10, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/46753f832d53430a9e8c5621eadd8ede, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/65a8a78854f340f7b4c6d253633ff9b7, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/0d83833f0cc6409aaef32788d6b008e4] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp, totalSize=53.7 K 2024-11-16T10:24:52,030 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/5b68f06626db45c9bf2da72332508d3a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/7ff10646cb4f4111afc4d186044b8e34, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/7f64b0b2c30d48d889888270a77a4730, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/721baee8f1624cf4ac4f032c20745ebb] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp, totalSize=46.9 K 2024-11-16T10:24:52,030 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b68f06626db45c9bf2da72332508d3a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731752688434 2024-11-16T10:24:52,030 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 079c5f79d974494f962ca2ab87ff4e10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731752688434 2024-11-16T10:24:52,031 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ff10646cb4f4111afc4d186044b8e34, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731752688464 2024-11-16T10:24:52,031 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46753f832d53430a9e8c5621eadd8ede, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731752688455 2024-11-16T10:24:52,031 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f64b0b2c30d48d889888270a77a4730, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731752689107 2024-11-16T10:24:52,031 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65a8a78854f340f7b4c6d253633ff9b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731752689107 2024-11-16T10:24:52,031 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 721baee8f1624cf4ac4f032c20745ebb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731752690335 2024-11-16T10:24:52,031 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d83833f0cc6409aaef32788d6b008e4, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731752690335 2024-11-16T10:24:52,040 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d41ecfe9dd6b68876325d42387a8c316#B#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:52,040 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/d6c3ecae3a7d4d7ea3e482eb579f64d3 is 50, key is test_row_0/B:col10/1731752691478/Put/seqid=0 2024-11-16T10:24:52,042 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d41ecfe9dd6b68876325d42387a8c316#A#compaction#175 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:52,043 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/2c51c0881613426684a4a9efeb9e3f49 is 50, key is test_row_0/A:col10/1731752691478/Put/seqid=0 2024-11-16T10:24:52,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742037_1213 (size=12139) 2024-11-16T10:24:52,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742036_1212 (size=12139) 2024-11-16T10:24:52,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:24:52,138 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:24:52,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d41ecfe9dd6b68876325d42387a8c316 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T10:24:52,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/4cd7cc0d896444069396ec33c4578bac is 50, key is test_row_0/A:col10/1731752692136/Put/seqid=0 2024-11-16T10:24:52,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742038_1214 (size=14341) 2024-11-16T10:24:52,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752752187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:52,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752752188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:52,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34482 deadline: 1731752752189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:52,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752752192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:52,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:52,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:52,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752752297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752752297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:52,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:52,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:52,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:52,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752752306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:52,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:52,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:52,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34482 deadline: 1731752752306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:52,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:52,457 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/2c51c0881613426684a4a9efeb9e3f49 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/2c51c0881613426684a4a9efeb9e3f49 2024-11-16T10:24:52,458 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/d6c3ecae3a7d4d7ea3e482eb579f64d3 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/d6c3ecae3a7d4d7ea3e482eb579f64d3 2024-11-16T10:24:52,465 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d41ecfe9dd6b68876325d42387a8c316/A of d41ecfe9dd6b68876325d42387a8c316 into 2c51c0881613426684a4a9efeb9e3f49(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:52,465 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:24:52,465 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., storeName=d41ecfe9dd6b68876325d42387a8c316/A, priority=12, startTime=1731752692028; duration=0sec 2024-11-16T10:24:52,465 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:24:52,465 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d41ecfe9dd6b68876325d42387a8c316:A 2024-11-16T10:24:52,465 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:24:52,467 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:24:52,467 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): d41ecfe9dd6b68876325d42387a8c316/C is initiating minor compaction (all files) 2024-11-16T10:24:52,467 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d41ecfe9dd6b68876325d42387a8c316/C in TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:52,467 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/eafbc0a753484a38b62bfbab850c5317, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/9615d03983c64ead93a5add93495ae12, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/cb0a202d233b48fd85a3911f0e19191e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/a7d3c98134794f0abd6f182178564d1c] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp, totalSize=46.9 K 2024-11-16T10:24:52,467 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting eafbc0a753484a38b62bfbab850c5317, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731752688434 2024-11-16T10:24:52,468 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9615d03983c64ead93a5add93495ae12, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731752688464 2024-11-16T10:24:52,468 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb0a202d233b48fd85a3911f0e19191e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731752689107 2024-11-16T10:24:52,468 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d41ecfe9dd6b68876325d42387a8c316/B of d41ecfe9dd6b68876325d42387a8c316 into d6c3ecae3a7d4d7ea3e482eb579f64d3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:52,468 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:24:52,468 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7d3c98134794f0abd6f182178564d1c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731752690335 2024-11-16T10:24:52,469 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., storeName=d41ecfe9dd6b68876325d42387a8c316/B, priority=12, startTime=1731752692028; duration=0sec 2024-11-16T10:24:52,469 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:52,469 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d41ecfe9dd6b68876325d42387a8c316:B 2024-11-16T10:24:52,478 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d41ecfe9dd6b68876325d42387a8c316#C#compaction#177 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:52,479 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/27ff40ddcd3249c7b7d0eaffa6bc3520 is 50, key is test_row_0/C:col10/1731752691478/Put/seqid=0 2024-11-16T10:24:52,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752752507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:52,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:52,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752752507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:52,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:52,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:52,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752752516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:52,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:52,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34482 deadline: 1731752752516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:52,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:52,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752752526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:52,531 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_1' in d41ecfe9dd6b68876325d42387a8c316 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:24:52,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:52,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742039_1215 (size=12139) 2024-11-16T10:24:52,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/4cd7cc0d896444069396ec33c4578bac 2024-11-16T10:24:52,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/d1a1b27513ba4568836a2fb1b3964987 is 50, key is test_row_0/B:col10/1731752692136/Put/seqid=0 2024-11-16T10:24:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T10:24:52,587 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:24:52,587 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:24:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-16T10:24:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T10:24:52,590 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:24:52,591 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:24:52,591 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:24:52,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742040_1216 (size=12001) 2024-11-16T10:24:52,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/d1a1b27513ba4568836a2fb1b3964987 2024-11-16T10:24:52,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/fb8ab969a5e64328958c0e172680696f is 50, key is test_row_0/C:col10/1731752692136/Put/seqid=0 2024-11-16T10:24:52,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T10:24:52,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742041_1217 (size=12001) 2024-11-16T10:24:52,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T10:24:52,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:52,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:52,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:52,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:52,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:52,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:52,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752752817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:52,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:52,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:52,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752752817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:52,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:52,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34482 deadline: 1731752752826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:52,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752752826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:52,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:52,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T10:24:52,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:52,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:52,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:52,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:52,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:52,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T10:24:52,948 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/27ff40ddcd3249c7b7d0eaffa6bc3520 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/27ff40ddcd3249c7b7d0eaffa6bc3520 2024-11-16T10:24:52,965 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d41ecfe9dd6b68876325d42387a8c316/C of d41ecfe9dd6b68876325d42387a8c316 into 27ff40ddcd3249c7b7d0eaffa6bc3520(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:52,965 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:24:52,965 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., storeName=d41ecfe9dd6b68876325d42387a8c316/C, priority=12, startTime=1731752692028; duration=0sec 2024-11-16T10:24:52,965 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:52,965 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d41ecfe9dd6b68876325d42387a8c316:C 2024-11-16T10:24:53,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T10:24:53,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:53,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:53,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:53,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:53,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:53,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:53,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/fb8ab969a5e64328958c0e172680696f 2024-11-16T10:24:53,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/4cd7cc0d896444069396ec33c4578bac as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/4cd7cc0d896444069396ec33c4578bac 2024-11-16T10:24:53,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/4cd7cc0d896444069396ec33c4578bac, entries=200, sequenceid=90, filesize=14.0 K 2024-11-16T10:24:53,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/d1a1b27513ba4568836a2fb1b3964987 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/d1a1b27513ba4568836a2fb1b3964987 2024-11-16T10:24:53,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/d1a1b27513ba4568836a2fb1b3964987, entries=150, sequenceid=90, filesize=11.7 K 2024-11-16T10:24:53,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/fb8ab969a5e64328958c0e172680696f as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/fb8ab969a5e64328958c0e172680696f 2024-11-16T10:24:53,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/fb8ab969a5e64328958c0e172680696f, entries=150, sequenceid=90, filesize=11.7 K 2024-11-16T10:24:53,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for d41ecfe9dd6b68876325d42387a8c316 in 988ms, sequenceid=90, compaction requested=false 2024-11-16T10:24:53,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:24:53,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T10:24:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:53,203 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing d41ecfe9dd6b68876325d42387a8c316 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T10:24:53,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/842f97f962044168aeb97cc499dd56e4 is 50, key is test_row_0/A:col10/1731752692184/Put/seqid=0 2024-11-16T10:24:53,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742042_1218 (size=12001) 2024-11-16T10:24:53,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T10:24:53,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:24:53,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:53,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:53,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34482 deadline: 1731752753336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:53,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:53,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752753337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:53,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:53,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:53,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:53,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752753339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:53,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752753339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:53,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:53,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752753446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:53,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:53,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:53,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:53,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:53,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752753446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:53,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:53,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:53,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:53,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:53,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752753447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:53,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:53,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:53,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:53,612 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/842f97f962044168aeb97cc499dd56e4 2024-11-16T10:24:53,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/a935947dbda041fc8f9858252c3fcf11 is 50, key is test_row_0/B:col10/1731752692184/Put/seqid=0 2024-11-16T10:24:53,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742043_1219 (size=12001) 2024-11-16T10:24:53,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:53,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752753656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:53,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:53,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752753656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:53,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:53,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:53,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:53,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752753657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:53,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:53,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:53,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:53,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:53,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T10:24:53,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:53,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752753966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:53,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:53,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752753966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:53,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752753966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:54,026 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/a935947dbda041fc8f9858252c3fcf11 2024-11-16T10:24:54,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/145f63603d414a548e1572ab5132b06f is 50, key is test_row_0/C:col10/1731752692184/Put/seqid=0 2024-11-16T10:24:54,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742044_1220 (size=12001) 2024-11-16T10:24:54,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34482 deadline: 1731752754346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:54,350 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_1' in d41ecfe9dd6b68876325d42387a8c316 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:24:54,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:54,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:54,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:54,448 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/145f63603d414a548e1572ab5132b06f 2024-11-16T10:24:54,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/842f97f962044168aeb97cc499dd56e4 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/842f97f962044168aeb97cc499dd56e4 2024-11-16T10:24:54,458 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/842f97f962044168aeb97cc499dd56e4, entries=150, sequenceid=117, filesize=11.7 K 2024-11-16T10:24:54,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/a935947dbda041fc8f9858252c3fcf11 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/a935947dbda041fc8f9858252c3fcf11 2024-11-16T10:24:54,473 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/a935947dbda041fc8f9858252c3fcf11, entries=150, sequenceid=117, filesize=11.7 K 2024-11-16T10:24:54,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/145f63603d414a548e1572ab5132b06f as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/145f63603d414a548e1572ab5132b06f 2024-11-16T10:24:54,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752754476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:54,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752754476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:54,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752754476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:54,484 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/145f63603d414a548e1572ab5132b06f, entries=150, sequenceid=117, filesize=11.7 K 2024-11-16T10:24:54,488 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for d41ecfe9dd6b68876325d42387a8c316 in 1285ms, sequenceid=117, compaction requested=true 2024-11-16T10:24:54,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:24:54,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:54,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-16T10:24:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-16T10:24:54,493 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-16T10:24:54,493 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8990 sec 2024-11-16T10:24:54,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.9060 sec 2024-11-16T10:24:54,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:24:54,560 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:24:54,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d41ecfe9dd6b68876325d42387a8c316 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T10:24:54,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/0c28b96c01a24b4398f669fcd913a715 is 50, key is test_row_0/A:col10/1731752693338/Put/seqid=0 2024-11-16T10:24:54,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742045_1221 (size=12051) 2024-11-16T10:24:54,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752754649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:54,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:54,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:54,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T10:24:54,737 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:24:54,738 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:24:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-16T10:24:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T10:24:54,740 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:24:54,740 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:24:54,740 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:24:54,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752754766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T10:24:54,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T10:24:54,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:54,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:54,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:54,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:54,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:54,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/0c28b96c01a24b4398f669fcd913a715 2024-11-16T10:24:54,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752754976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:54,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/af62138b27c7443996787c342a217f53 is 50, key is test_row_0/B:col10/1731752693338/Put/seqid=0 2024-11-16T10:24:54,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:54,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:54,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:54,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742046_1222 (size=12051) 2024-11-16T10:24:55,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T10:24:55,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:55,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:55,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:55,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:55,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T10:24:55,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T10:24:55,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:55,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:55,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:55,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:55,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:55,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:55,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:55,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752755286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:55,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:55,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:55,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:55,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T10:24:55,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:55,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:55,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:55,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:55,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:55,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:55,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T10:24:55,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/af62138b27c7443996787c342a217f53 2024-11-16T10:24:55,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/8d05420950994a03badd5bdf5be6a450 is 50, key is test_row_0/C:col10/1731752693338/Put/seqid=0 2024-11-16T10:24:55,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742047_1223 (size=12051) 2024-11-16T10:24:55,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/8d05420950994a03badd5bdf5be6a450 2024-11-16T10:24:55,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/0c28b96c01a24b4398f669fcd913a715 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/0c28b96c01a24b4398f669fcd913a715 2024-11-16T10:24:55,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/0c28b96c01a24b4398f669fcd913a715, entries=150, sequenceid=130, filesize=11.8 K 2024-11-16T10:24:55,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/af62138b27c7443996787c342a217f53 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/af62138b27c7443996787c342a217f53 2024-11-16T10:24:55,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/af62138b27c7443996787c342a217f53, entries=150, sequenceid=130, filesize=11.8 K 2024-11-16T10:24:55,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/8d05420950994a03badd5bdf5be6a450 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/8d05420950994a03badd5bdf5be6a450 2024-11-16T10:24:55,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/8d05420950994a03badd5bdf5be6a450, entries=150, sequenceid=130, filesize=11.8 K 2024-11-16T10:24:55,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d41ecfe9dd6b68876325d42387a8c316 in 855ms, sequenceid=130, compaction requested=true 2024-11-16T10:24:55,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:24:55,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d41ecfe9dd6b68876325d42387a8c316:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:24:55,416 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:24:55,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:55,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d41ecfe9dd6b68876325d42387a8c316:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:24:55,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:55,416 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:24:55,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d41ecfe9dd6b68876325d42387a8c316:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:24:55,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:24:55,417 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:24:55,417 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:24:55,417 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): d41ecfe9dd6b68876325d42387a8c316/B is initiating minor compaction (all files) 2024-11-16T10:24:55,417 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): d41ecfe9dd6b68876325d42387a8c316/A is initiating minor compaction (all files) 2024-11-16T10:24:55,417 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d41ecfe9dd6b68876325d42387a8c316/B in TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:55,417 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d41ecfe9dd6b68876325d42387a8c316/A in TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:55,417 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/d6c3ecae3a7d4d7ea3e482eb579f64d3, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/d1a1b27513ba4568836a2fb1b3964987, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/a935947dbda041fc8f9858252c3fcf11, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/af62138b27c7443996787c342a217f53] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp, totalSize=47.1 K 2024-11-16T10:24:55,417 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/2c51c0881613426684a4a9efeb9e3f49, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/4cd7cc0d896444069396ec33c4578bac, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/842f97f962044168aeb97cc499dd56e4, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/0c28b96c01a24b4398f669fcd913a715] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp, totalSize=49.3 K 2024-11-16T10:24:55,417 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting d6c3ecae3a7d4d7ea3e482eb579f64d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731752690335 2024-11-16T10:24:55,417 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c51c0881613426684a4a9efeb9e3f49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731752690335 2024-11-16T10:24:55,418 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting d1a1b27513ba4568836a2fb1b3964987, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731752692136 2024-11-16T10:24:55,418 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cd7cc0d896444069396ec33c4578bac, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731752691500 2024-11-16T10:24:55,418 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting a935947dbda041fc8f9858252c3fcf11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731752692184 2024-11-16T10:24:55,418 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 842f97f962044168aeb97cc499dd56e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731752692184 2024-11-16T10:24:55,418 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting af62138b27c7443996787c342a217f53, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731752693336 2024-11-16T10:24:55,418 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c28b96c01a24b4398f669fcd913a715, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731752693336 2024-11-16T10:24:55,428 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d41ecfe9dd6b68876325d42387a8c316#A#compaction#187 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:55,428 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d41ecfe9dd6b68876325d42387a8c316#B#compaction#186 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:55,428 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/d10f0e533acc438f8fe9f6e865c2ea36 is 50, key is test_row_0/A:col10/1731752693338/Put/seqid=0 2024-11-16T10:24:55,428 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/b3ceedbf4dcb421e9f6110acdfef29b8 is 50, key is test_row_0/B:col10/1731752693338/Put/seqid=0 2024-11-16T10:24:55,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742049_1225 (size=12325) 2024-11-16T10:24:55,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742048_1224 (size=12325) 2024-11-16T10:24:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:24:55,489 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:24:55,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d41ecfe9dd6b68876325d42387a8c316 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T10:24:55,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/95bd5cd41ae04f26914a6ef4d860c417 is 50, key is test_row_0/A:col10/1731752694641/Put/seqid=0 2024-11-16T10:24:55,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742050_1226 (size=14541) 2024-11-16T10:24:55,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T10:24:55,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:55,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:55,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:55,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:55,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:55,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752755530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:55,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:55,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752755534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:55,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752755535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:55,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:55,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752755646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:55,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:55,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:55,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:55,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752755646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:55,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:55,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:55,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752755647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:55,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T10:24:55,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:55,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:55,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:55,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:55,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:55,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752755797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:55,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:55,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:55,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:55,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T10:24:55,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:55,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:55,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:55,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:55,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:55,839 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/d10f0e533acc438f8fe9f6e865c2ea36 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/d10f0e533acc438f8fe9f6e865c2ea36 2024-11-16T10:24:55,840 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/b3ceedbf4dcb421e9f6110acdfef29b8 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/b3ceedbf4dcb421e9f6110acdfef29b8 2024-11-16T10:24:55,852 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d41ecfe9dd6b68876325d42387a8c316/A of d41ecfe9dd6b68876325d42387a8c316 into d10f0e533acc438f8fe9f6e865c2ea36(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:55,852 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:24:55,852 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., storeName=d41ecfe9dd6b68876325d42387a8c316/A, priority=12, startTime=1731752695416; duration=0sec 2024-11-16T10:24:55,852 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:24:55,852 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d41ecfe9dd6b68876325d42387a8c316:A 2024-11-16T10:24:55,852 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:24:55,854 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d41ecfe9dd6b68876325d42387a8c316/B of d41ecfe9dd6b68876325d42387a8c316 into b3ceedbf4dcb421e9f6110acdfef29b8(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:55,854 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:24:55,854 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., storeName=d41ecfe9dd6b68876325d42387a8c316/B, priority=12, startTime=1731752695416; duration=0sec 2024-11-16T10:24:55,854 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:55,854 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d41ecfe9dd6b68876325d42387a8c316:B 2024-11-16T10:24:55,857 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:24:55,857 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): d41ecfe9dd6b68876325d42387a8c316/C is initiating minor compaction (all files) 2024-11-16T10:24:55,857 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d41ecfe9dd6b68876325d42387a8c316/C in TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:55,857 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/27ff40ddcd3249c7b7d0eaffa6bc3520, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/fb8ab969a5e64328958c0e172680696f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/145f63603d414a548e1572ab5132b06f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/8d05420950994a03badd5bdf5be6a450] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp, totalSize=47.1 K 2024-11-16T10:24:55,858 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27ff40ddcd3249c7b7d0eaffa6bc3520, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731752690335 2024-11-16T10:24:55,858 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb8ab969a5e64328958c0e172680696f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731752692136 2024-11-16T10:24:55,858 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 145f63603d414a548e1572ab5132b06f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731752692184 2024-11-16T10:24:55,859 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d05420950994a03badd5bdf5be6a450, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731752693336 2024-11-16T10:24:55,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752755856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:55,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:55,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:55,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:55,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752755856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:55,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752755856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:55,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:55,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:55,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:55,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:55,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:55,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:55,870 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d41ecfe9dd6b68876325d42387a8c316#C#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:55,871 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/a35810f51bbc4ef88da4060895b5ea80 is 50, key is test_row_0/C:col10/1731752693338/Put/seqid=0 2024-11-16T10:24:55,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T10:24:55,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742051_1227 (size=12325) 2024-11-16T10:24:55,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/95bd5cd41ae04f26914a6ef4d860c417 2024-11-16T10:24:55,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/5b3c26cbf8544a73bcb5a1ddebfd58f8 is 50, key is test_row_0/B:col10/1731752694641/Put/seqid=0 2024-11-16T10:24:55,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742052_1228 (size=12151) 2024-11-16T10:24:55,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T10:24:55,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:55,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:55,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:55,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:55,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:56,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T10:24:56,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:56,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:56,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:56,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:56,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:56,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:56,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752756166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:56,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:56,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:56,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:56,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:56,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752756167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:56,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:56,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:56,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:56,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:56,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752756176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:56,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:56,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:56,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:56,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T10:24:56,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:56,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:56,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:56,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:56,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:56,304 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/a35810f51bbc4ef88da4060895b5ea80 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/a35810f51bbc4ef88da4060895b5ea80 2024-11-16T10:24:56,310 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d41ecfe9dd6b68876325d42387a8c316/C of d41ecfe9dd6b68876325d42387a8c316 into a35810f51bbc4ef88da4060895b5ea80(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:56,310 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:24:56,310 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., storeName=d41ecfe9dd6b68876325d42387a8c316/C, priority=12, startTime=1731752695416; duration=0sec 2024-11-16T10:24:56,310 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:56,311 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d41ecfe9dd6b68876325d42387a8c316:C 2024-11-16T10:24:56,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/5b3c26cbf8544a73bcb5a1ddebfd58f8 2024-11-16T10:24:56,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/04b6b0d40a4a450593ea3be60dc6749e is 50, key is test_row_0/C:col10/1731752694641/Put/seqid=0 2024-11-16T10:24:56,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742053_1229 (size=12151) 2024-11-16T10:24:56,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34482 deadline: 1731752756357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:56,360 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_1' in d41ecfe9dd6b68876325d42387a8c316 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:24:56,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:56,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:56,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:56,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T10:24:56,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:56,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:56,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:56,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:56,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:56,495 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T10:24:56,495 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-16T10:24:56,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T10:24:56,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:56,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:56,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:56,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:56,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:56,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:56,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752756676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:56,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752756677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:56,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:56,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752756686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:56,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:56,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:56,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T10:24:56,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:56,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:56,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:56,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:56,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:56,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:56,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/04b6b0d40a4a450593ea3be60dc6749e 2024-11-16T10:24:56,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/95bd5cd41ae04f26914a6ef4d860c417 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/95bd5cd41ae04f26914a6ef4d860c417 2024-11-16T10:24:56,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/95bd5cd41ae04f26914a6ef4d860c417, entries=200, sequenceid=155, filesize=14.2 K 2024-11-16T10:24:56,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/5b3c26cbf8544a73bcb5a1ddebfd58f8 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/5b3c26cbf8544a73bcb5a1ddebfd58f8 2024-11-16T10:24:56,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/5b3c26cbf8544a73bcb5a1ddebfd58f8, entries=150, sequenceid=155, filesize=11.9 K 2024-11-16T10:24:56,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/04b6b0d40a4a450593ea3be60dc6749e as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/04b6b0d40a4a450593ea3be60dc6749e 2024-11-16T10:24:56,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/04b6b0d40a4a450593ea3be60dc6749e, entries=150, sequenceid=155, filesize=11.9 K 2024-11-16T10:24:56,786 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for d41ecfe9dd6b68876325d42387a8c316 in 1297ms, sequenceid=155, compaction requested=false 2024-11-16T10:24:56,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:24:56,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:24:56,819 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:24:56,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d41ecfe9dd6b68876325d42387a8c316 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T10:24:56,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/ce45b4528e6142c58557482caf8f0509 is 50, key is test_row_0/A:col10/1731752695534/Put/seqid=0 2024-11-16T10:24:56,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742054_1230 (size=12151) 2024-11-16T10:24:56,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/ce45b4528e6142c58557482caf8f0509 2024-11-16T10:24:56,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/bf5f5dcba33e4e30a21c1102581ddfed is 50, key is test_row_0/B:col10/1731752695534/Put/seqid=0 2024-11-16T10:24:56,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742055_1231 (size=12151) 2024-11-16T10:24:56,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T10:24:56,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T10:24:56,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:56,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:56,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:56,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:56,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:56,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:56,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:56,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752756909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:56,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:56,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:56,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:57,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:57,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752757017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:57,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:57,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:57,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:57,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T10:24:57,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:57,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:57,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:57,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:57,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:57,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T10:24:57,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:57,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:57,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:57,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:57,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:57,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752757226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:57,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:57,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:57,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:57,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/bf5f5dcba33e4e30a21c1102581ddfed 2024-11-16T10:24:57,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/40ee0f215787447ea2f5ef6546b6ddef is 50, key is test_row_0/C:col10/1731752695534/Put/seqid=0 2024-11-16T10:24:57,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742056_1232 (size=12151) 2024-11-16T10:24:57,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/40ee0f215787447ea2f5ef6546b6ddef 2024-11-16T10:24:57,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/ce45b4528e6142c58557482caf8f0509 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/ce45b4528e6142c58557482caf8f0509 2024-11-16T10:24:57,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/ce45b4528e6142c58557482caf8f0509, entries=150, sequenceid=170, filesize=11.9 K 2024-11-16T10:24:57,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/bf5f5dcba33e4e30a21c1102581ddfed as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/bf5f5dcba33e4e30a21c1102581ddfed 2024-11-16T10:24:57,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/bf5f5dcba33e4e30a21c1102581ddfed, entries=150, sequenceid=170, filesize=11.9 K 2024-11-16T10:24:57,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/40ee0f215787447ea2f5ef6546b6ddef as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/40ee0f215787447ea2f5ef6546b6ddef 2024-11-16T10:24:57,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/40ee0f215787447ea2f5ef6546b6ddef, entries=150, sequenceid=170, filesize=11.9 K 2024-11-16T10:24:57,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for d41ecfe9dd6b68876325d42387a8c316 in 525ms, sequenceid=170, compaction requested=true 2024-11-16T10:24:57,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:24:57,345 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:24:57,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d41ecfe9dd6b68876325d42387a8c316:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:24:57,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:57,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d41ecfe9dd6b68876325d42387a8c316:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:24:57,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:57,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d41ecfe9dd6b68876325d42387a8c316:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:24:57,345 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:24:57,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:24:57,346 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39017 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:24:57,346 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): d41ecfe9dd6b68876325d42387a8c316/A is initiating minor compaction (all files) 2024-11-16T10:24:57,346 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d41ecfe9dd6b68876325d42387a8c316/A in TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:57,346 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/d10f0e533acc438f8fe9f6e865c2ea36, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/95bd5cd41ae04f26914a6ef4d860c417, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/ce45b4528e6142c58557482caf8f0509] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp, totalSize=38.1 K 2024-11-16T10:24:57,347 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:24:57,347 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting d10f0e533acc438f8fe9f6e865c2ea36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731752693336 2024-11-16T10:24:57,347 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): d41ecfe9dd6b68876325d42387a8c316/B is initiating minor compaction (all files) 2024-11-16T10:24:57,347 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d41ecfe9dd6b68876325d42387a8c316/B in TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:57,347 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/b3ceedbf4dcb421e9f6110acdfef29b8, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/5b3c26cbf8544a73bcb5a1ddebfd58f8, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/bf5f5dcba33e4e30a21c1102581ddfed] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp, totalSize=35.8 K 2024-11-16T10:24:57,347 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95bd5cd41ae04f26914a6ef4d860c417, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731752694630 2024-11-16T10:24:57,347 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting b3ceedbf4dcb421e9f6110acdfef29b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731752693336 2024-11-16T10:24:57,348 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce45b4528e6142c58557482caf8f0509, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731752695529 2024-11-16T10:24:57,348 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b3c26cbf8544a73bcb5a1ddebfd58f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731752694630 2024-11-16T10:24:57,349 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting bf5f5dcba33e4e30a21c1102581ddfed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731752695529 2024-11-16T10:24:57,370 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d41ecfe9dd6b68876325d42387a8c316#A#compaction#195 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:57,371 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/5fd9b2751d8a466e87dc10e82c2e65f9 is 50, key is test_row_0/A:col10/1731752695534/Put/seqid=0 2024-11-16T10:24:57,382 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d41ecfe9dd6b68876325d42387a8c316#B#compaction#196 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:57,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T10:24:57,383 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/38e42773d43a41ef9cc4d4bb5efe4951 is 50, key is test_row_0/B:col10/1731752695534/Put/seqid=0 2024-11-16T10:24:57,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:57,383 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing d41ecfe9dd6b68876325d42387a8c316 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T10:24:57,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/824d1832f5cc4cc49fbdf232955689ba is 50, key is test_row_0/A:col10/1731752696907/Put/seqid=0 2024-11-16T10:24:57,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742057_1233 (size=12527) 2024-11-16T10:24:57,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742058_1234 (size=12527) 2024-11-16T10:24:57,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742059_1235 (size=12151) 2024-11-16T10:24:57,464 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/824d1832f5cc4cc49fbdf232955689ba 2024-11-16T10:24:57,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/5edfc535335245179c12b1056c7af95e is 50, key is test_row_0/B:col10/1731752696907/Put/seqid=0 2024-11-16T10:24:57,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742060_1236 (size=12151) 2024-11-16T10:24:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:24:57,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:57,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:57,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752757580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:57,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:57,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:57,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:57,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:57,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752757686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:57,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:57,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752757686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:57,691 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_2' in d41ecfe9dd6b68876325d42387a8c316 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:24:57,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:57,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:57,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:57,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:57,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:57,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:57,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:57,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752757697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:57,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:57,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752757697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:57,701 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_0' in d41ecfe9dd6b68876325d42387a8c316 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:24:57,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:57,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:57,701 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_2' in d41ecfe9dd6b68876325d42387a8c316 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:24:57,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:57,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:57,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:57,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:57,824 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/5fd9b2751d8a466e87dc10e82c2e65f9 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/5fd9b2751d8a466e87dc10e82c2e65f9 2024-11-16T10:24:57,828 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d41ecfe9dd6b68876325d42387a8c316/A of d41ecfe9dd6b68876325d42387a8c316 into 5fd9b2751d8a466e87dc10e82c2e65f9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:57,828 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:24:57,828 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., storeName=d41ecfe9dd6b68876325d42387a8c316/A, priority=13, startTime=1731752697344; duration=0sec 2024-11-16T10:24:57,828 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:24:57,828 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d41ecfe9dd6b68876325d42387a8c316:A 2024-11-16T10:24:57,828 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:24:57,829 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:24:57,829 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): d41ecfe9dd6b68876325d42387a8c316/C is initiating minor compaction (all files) 2024-11-16T10:24:57,829 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d41ecfe9dd6b68876325d42387a8c316/C in TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:57,830 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/a35810f51bbc4ef88da4060895b5ea80, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/04b6b0d40a4a450593ea3be60dc6749e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/40ee0f215787447ea2f5ef6546b6ddef] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp, totalSize=35.8 K 2024-11-16T10:24:57,830 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting a35810f51bbc4ef88da4060895b5ea80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731752693336 2024-11-16T10:24:57,830 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04b6b0d40a4a450593ea3be60dc6749e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731752694630 2024-11-16T10:24:57,830 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40ee0f215787447ea2f5ef6546b6ddef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731752695529 2024-11-16T10:24:57,838 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/38e42773d43a41ef9cc4d4bb5efe4951 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/38e42773d43a41ef9cc4d4bb5efe4951 2024-11-16T10:24:57,839 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d41ecfe9dd6b68876325d42387a8c316#C#compaction#199 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:57,839 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/0351adc88b994c80825f2e817c327573 is 50, key is test_row_0/C:col10/1731752695534/Put/seqid=0 2024-11-16T10:24:57,843 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d41ecfe9dd6b68876325d42387a8c316/B of d41ecfe9dd6b68876325d42387a8c316 into 38e42773d43a41ef9cc4d4bb5efe4951(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:57,843 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:24:57,843 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., storeName=d41ecfe9dd6b68876325d42387a8c316/B, priority=13, startTime=1731752697345; duration=0sec 2024-11-16T10:24:57,843 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:57,843 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d41ecfe9dd6b68876325d42387a8c316:B 2024-11-16T10:24:57,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742061_1237 (size=12527) 2024-11-16T10:24:57,854 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/0351adc88b994c80825f2e817c327573 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/0351adc88b994c80825f2e817c327573 2024-11-16T10:24:57,861 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d41ecfe9dd6b68876325d42387a8c316/C of d41ecfe9dd6b68876325d42387a8c316 into 0351adc88b994c80825f2e817c327573(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:57,862 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:24:57,862 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., storeName=d41ecfe9dd6b68876325d42387a8c316/C, priority=13, startTime=1731752697345; duration=0sec 2024-11-16T10:24:57,862 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:57,862 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d41ecfe9dd6b68876325d42387a8c316:C 2024-11-16T10:24:57,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:57,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752757896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:57,902 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/5edfc535335245179c12b1056c7af95e 2024-11-16T10:24:57,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/e72b6ccfc85642c6bde751c3f2fc08b1 is 50, key is test_row_0/C:col10/1731752696907/Put/seqid=0 2024-11-16T10:24:57,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742062_1238 (size=12151) 2024-11-16T10:24:57,930 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/e72b6ccfc85642c6bde751c3f2fc08b1 2024-11-16T10:24:57,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/824d1832f5cc4cc49fbdf232955689ba as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/824d1832f5cc4cc49fbdf232955689ba 2024-11-16T10:24:57,949 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/824d1832f5cc4cc49fbdf232955689ba, entries=150, sequenceid=194, filesize=11.9 K 2024-11-16T10:24:57,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/5edfc535335245179c12b1056c7af95e as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/5edfc535335245179c12b1056c7af95e 2024-11-16T10:24:57,957 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/5edfc535335245179c12b1056c7af95e, entries=150, sequenceid=194, filesize=11.9 K 2024-11-16T10:24:57,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/e72b6ccfc85642c6bde751c3f2fc08b1 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/e72b6ccfc85642c6bde751c3f2fc08b1 2024-11-16T10:24:57,964 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/e72b6ccfc85642c6bde751c3f2fc08b1, entries=150, sequenceid=194, filesize=11.9 K 2024-11-16T10:24:57,965 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for d41ecfe9dd6b68876325d42387a8c316 in 582ms, sequenceid=194, compaction requested=false 2024-11-16T10:24:57,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:24:57,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:57,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-16T10:24:57,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-16T10:24:57,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-16T10:24:57,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2260 sec 2024-11-16T10:24:57,970 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 3.2310 sec 2024-11-16T10:24:58,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:24:58,208 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:24:58,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d41ecfe9dd6b68876325d42387a8c316 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T10:24:58,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/176cc05b25be453aa2053e54bb29aa56 is 50, key is test_row_0/A:col10/1731752697577/Put/seqid=0 2024-11-16T10:24:58,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742063_1239 (size=14541) 2024-11-16T10:24:58,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/176cc05b25be453aa2053e54bb29aa56 2024-11-16T10:24:58,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/e77beacbb74245d191a9292381cb941e is 50, key is test_row_0/B:col10/1731752697577/Put/seqid=0 2024-11-16T10:24:58,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742064_1240 (size=12151) 2024-11-16T10:24:58,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/e77beacbb74245d191a9292381cb941e 2024-11-16T10:24:58,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/a9a67a2de22d41b4a7d9c4b6f640c635 is 50, key is test_row_0/C:col10/1731752697577/Put/seqid=0 2024-11-16T10:24:58,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742065_1241 (size=12151) 2024-11-16T10:24:58,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/a9a67a2de22d41b4a7d9c4b6f640c635 2024-11-16T10:24:58,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/176cc05b25be453aa2053e54bb29aa56 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/176cc05b25be453aa2053e54bb29aa56 2024-11-16T10:24:58,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/176cc05b25be453aa2053e54bb29aa56, entries=200, sequenceid=210, filesize=14.2 K 2024-11-16T10:24:58,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/e77beacbb74245d191a9292381cb941e as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/e77beacbb74245d191a9292381cb941e 2024-11-16T10:24:58,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/e77beacbb74245d191a9292381cb941e, entries=150, sequenceid=210, filesize=11.9 K 2024-11-16T10:24:58,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/a9a67a2de22d41b4a7d9c4b6f640c635 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/a9a67a2de22d41b4a7d9c4b6f640c635 2024-11-16T10:24:58,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/a9a67a2de22d41b4a7d9c4b6f640c635, entries=150, sequenceid=210, filesize=11.9 K 2024-11-16T10:24:58,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=107.34 KB/109920 for d41ecfe9dd6b68876325d42387a8c316 in 82ms, sequenceid=210, compaction requested=true 2024-11-16T10:24:58,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:24:58,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d41ecfe9dd6b68876325d42387a8c316:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:24:58,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:58,291 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:24:58,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d41ecfe9dd6b68876325d42387a8c316:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:24:58,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:58,291 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:24:58,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d41ecfe9dd6b68876325d42387a8c316:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:24:58,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:24:58,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:24:58,292 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:24:58,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d41ecfe9dd6b68876325d42387a8c316 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-16T10:24:58,293 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:24:58,293 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:24:58,293 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): d41ecfe9dd6b68876325d42387a8c316/B is initiating minor compaction (all files) 2024-11-16T10:24:58,293 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): d41ecfe9dd6b68876325d42387a8c316/A is initiating minor compaction (all files) 2024-11-16T10:24:58,293 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d41ecfe9dd6b68876325d42387a8c316/A in TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:58,293 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d41ecfe9dd6b68876325d42387a8c316/B in TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:58,293 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/38e42773d43a41ef9cc4d4bb5efe4951, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/5edfc535335245179c12b1056c7af95e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/e77beacbb74245d191a9292381cb941e] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp, totalSize=36.0 K 2024-11-16T10:24:58,293 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/5fd9b2751d8a466e87dc10e82c2e65f9, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/824d1832f5cc4cc49fbdf232955689ba, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/176cc05b25be453aa2053e54bb29aa56] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp, totalSize=38.3 K 2024-11-16T10:24:58,294 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 38e42773d43a41ef9cc4d4bb5efe4951, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731752695529 2024-11-16T10:24:58,294 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fd9b2751d8a466e87dc10e82c2e65f9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731752695529 2024-11-16T10:24:58,294 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 5edfc535335245179c12b1056c7af95e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731752696901 2024-11-16T10:24:58,295 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 824d1832f5cc4cc49fbdf232955689ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731752696901 2024-11-16T10:24:58,295 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting e77beacbb74245d191a9292381cb941e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731752697567 2024-11-16T10:24:58,295 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 176cc05b25be453aa2053e54bb29aa56, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731752697567 2024-11-16T10:24:58,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/295ea4eae6714371a53eb51bd8fc7e0f is 50, key is test_row_0/A:col10/1731752698257/Put/seqid=0 2024-11-16T10:24:58,306 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d41ecfe9dd6b68876325d42387a8c316#B#compaction#206 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:58,306 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d41ecfe9dd6b68876325d42387a8c316#A#compaction#205 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:58,307 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/23f506fdf4af403bb57dac4d0f30732a is 50, key is test_row_0/A:col10/1731752697577/Put/seqid=0 2024-11-16T10:24:58,307 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/80d592bfe9a24821ad9f2e545fe129e6 is 50, key is test_row_0/B:col10/1731752697577/Put/seqid=0 2024-11-16T10:24:58,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742066_1242 (size=14541) 2024-11-16T10:24:58,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742067_1243 (size=12629) 2024-11-16T10:24:58,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742068_1244 (size=12629) 2024-11-16T10:24:58,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752758358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:58,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:58,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:58,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:58,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:58,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752758466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:58,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:58,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:58,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:58,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:58,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752758676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:58,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:58,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:58,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:58,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/295ea4eae6714371a53eb51bd8fc7e0f 2024-11-16T10:24:58,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/abb4e6a18003462c9319b104e8467384 is 50, key is test_row_0/B:col10/1731752698257/Put/seqid=0 2024-11-16T10:24:58,764 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/23f506fdf4af403bb57dac4d0f30732a as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/23f506fdf4af403bb57dac4d0f30732a 2024-11-16T10:24:58,764 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/80d592bfe9a24821ad9f2e545fe129e6 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/80d592bfe9a24821ad9f2e545fe129e6 2024-11-16T10:24:58,770 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d41ecfe9dd6b68876325d42387a8c316/A of d41ecfe9dd6b68876325d42387a8c316 into 23f506fdf4af403bb57dac4d0f30732a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:58,770 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d41ecfe9dd6b68876325d42387a8c316/B of d41ecfe9dd6b68876325d42387a8c316 into 80d592bfe9a24821ad9f2e545fe129e6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:58,770 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:24:58,770 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:24:58,770 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., storeName=d41ecfe9dd6b68876325d42387a8c316/B, priority=13, startTime=1731752698291; duration=0sec 2024-11-16T10:24:58,770 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., storeName=d41ecfe9dd6b68876325d42387a8c316/A, priority=13, startTime=1731752698291; duration=0sec 2024-11-16T10:24:58,770 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:24:58,770 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d41ecfe9dd6b68876325d42387a8c316:B 2024-11-16T10:24:58,770 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:24:58,770 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d41ecfe9dd6b68876325d42387a8c316:A 2024-11-16T10:24:58,770 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:24:58,772 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:24:58,772 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): d41ecfe9dd6b68876325d42387a8c316/C is initiating minor compaction (all files) 2024-11-16T10:24:58,772 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d41ecfe9dd6b68876325d42387a8c316/C in TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:58,772 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/0351adc88b994c80825f2e817c327573, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/e72b6ccfc85642c6bde751c3f2fc08b1, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/a9a67a2de22d41b4a7d9c4b6f640c635] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp, totalSize=36.0 K 2024-11-16T10:24:58,774 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 0351adc88b994c80825f2e817c327573, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731752695529 2024-11-16T10:24:58,774 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting e72b6ccfc85642c6bde751c3f2fc08b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731752696901 2024-11-16T10:24:58,774 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting a9a67a2de22d41b4a7d9c4b6f640c635, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731752697567 2024-11-16T10:24:58,787 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d41ecfe9dd6b68876325d42387a8c316#C#compaction#208 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:24:58,787 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/67df6bcd436f464ca15b8eff760cd5e3 is 50, key is test_row_0/C:col10/1731752697577/Put/seqid=0 2024-11-16T10:24:58,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742069_1245 (size=12151) 2024-11-16T10:24:58,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/abb4e6a18003462c9319b104e8467384 2024-11-16T10:24:58,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/b176345f0aa74a53a2a6f735849c7975 is 50, key is test_row_0/C:col10/1731752698257/Put/seqid=0 2024-11-16T10:24:58,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742070_1246 (size=12629) 2024-11-16T10:24:58,829 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/67df6bcd436f464ca15b8eff760cd5e3 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/67df6bcd436f464ca15b8eff760cd5e3 2024-11-16T10:24:58,836 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d41ecfe9dd6b68876325d42387a8c316/C of d41ecfe9dd6b68876325d42387a8c316 into 67df6bcd436f464ca15b8eff760cd5e3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:24:58,836 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:24:58,836 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., storeName=d41ecfe9dd6b68876325d42387a8c316/C, priority=13, startTime=1731752698291; duration=0sec 2024-11-16T10:24:58,836 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:24:58,836 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d41ecfe9dd6b68876325d42387a8c316:C 2024-11-16T10:24:58,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742071_1247 (size=12151) 2024-11-16T10:24:58,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T10:24:58,907 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:24:58,907 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:24:58,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-16T10:24:58,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T10:24:58,910 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:24:58,911 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:24:58,911 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:24:58,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:58,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752758986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:58,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:58,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:58,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:59,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T10:24:59,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T10:24:59,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:59,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:59,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:59,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:59,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:59,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:59,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T10:24:59,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:59,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:59,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:59,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:59,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:59,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:24:59,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T10:24:59,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/b176345f0aa74a53a2a6f735849c7975 2024-11-16T10:24:59,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/295ea4eae6714371a53eb51bd8fc7e0f as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/295ea4eae6714371a53eb51bd8fc7e0f 2024-11-16T10:24:59,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/295ea4eae6714371a53eb51bd8fc7e0f, entries=200, sequenceid=230, filesize=14.2 K 2024-11-16T10:24:59,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/abb4e6a18003462c9319b104e8467384 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/abb4e6a18003462c9319b104e8467384 2024-11-16T10:24:59,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/abb4e6a18003462c9319b104e8467384, entries=150, sequenceid=230, filesize=11.9 K 2024-11-16T10:24:59,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/b176345f0aa74a53a2a6f735849c7975 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/b176345f0aa74a53a2a6f735849c7975 2024-11-16T10:24:59,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/b176345f0aa74a53a2a6f735849c7975, entries=150, sequenceid=230, filesize=11.9 K 2024-11-16T10:24:59,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for d41ecfe9dd6b68876325d42387a8c316 in 996ms, sequenceid=230, compaction requested=false 2024-11-16T10:24:59,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:24:59,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T10:24:59,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:59,371 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing d41ecfe9dd6b68876325d42387a8c316 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-16T10:24:59,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/c6e70130c27e4a3b90580223cc06fdff is 50, key is test_row_0/A:col10/1731752698356/Put/seqid=0 2024-11-16T10:24:59,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742072_1248 (size=12151) 2024-11-16T10:24:59,427 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/c6e70130c27e4a3b90580223cc06fdff 2024-11-16T10:24:59,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/0f029ed5fe1a4f45bbd97c8380404cbe is 50, key is test_row_0/B:col10/1731752698356/Put/seqid=0 2024-11-16T10:24:59,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742073_1249 (size=12151) 2024-11-16T10:24:59,466 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/0f029ed5fe1a4f45bbd97c8380404cbe 2024-11-16T10:24:59,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/00c3e817ac1a41c0abf16ec3ecb79539 is 50, key is test_row_0/C:col10/1731752698356/Put/seqid=0 2024-11-16T10:24:59,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742074_1250 (size=12151) 2024-11-16T10:24:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:24:59,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:24:59,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T10:24:59,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752759568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:59,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:59,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:59,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:59,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:59,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752759686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:59,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:59,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:59,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:59,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:59,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752759706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:59,708 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_2' in d41ecfe9dd6b68876325d42387a8c316 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:24:59,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:59,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:59,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:59,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:59,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752759716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:59,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:59,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752759716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:59,720 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_0' in d41ecfe9dd6b68876325d42387a8c316 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:24:59,720 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_2' in d41ecfe9dd6b68876325d42387a8c316 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:24:59,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:59,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:59,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:59,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:59,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:59,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:59,886 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/00c3e817ac1a41c0abf16ec3ecb79539 2024-11-16T10:24:59,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/c6e70130c27e4a3b90580223cc06fdff as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/c6e70130c27e4a3b90580223cc06fdff 2024-11-16T10:24:59,894 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/c6e70130c27e4a3b90580223cc06fdff, entries=150, sequenceid=249, filesize=11.9 K 2024-11-16T10:24:59,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/0f029ed5fe1a4f45bbd97c8380404cbe as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/0f029ed5fe1a4f45bbd97c8380404cbe 2024-11-16T10:24:59,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:24:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752759896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:24:59,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:59,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:24:59,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:24:59,901 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/0f029ed5fe1a4f45bbd97c8380404cbe, entries=150, sequenceid=249, filesize=11.9 K 2024-11-16T10:24:59,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/00c3e817ac1a41c0abf16ec3ecb79539 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/00c3e817ac1a41c0abf16ec3ecb79539 2024-11-16T10:24:59,907 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/00c3e817ac1a41c0abf16ec3ecb79539, entries=150, sequenceid=249, filesize=11.9 K 2024-11-16T10:24:59,908 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for d41ecfe9dd6b68876325d42387a8c316 in 537ms, sequenceid=249, compaction requested=true 2024-11-16T10:24:59,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:24:59,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:24:59,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-16T10:24:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-16T10:24:59,912 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-16T10:24:59,912 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 999 msec 2024-11-16T10:24:59,915 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.0060 sec 2024-11-16T10:25:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T10:25:00,046 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:25:00,046 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:25:00,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-16T10:25:00,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T10:25:00,048 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:25:00,049 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:25:00,049 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:25:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T10:25:00,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T10:25:00,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:00,202 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing d41ecfe9dd6b68876325d42387a8c316 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-16T10:25:00,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/fe679ab2f6c846abb5ddbe0067e2e81e is 50, key is test_row_0/A:col10/1731752699566/Put/seqid=0 2024-11-16T10:25:00,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:25:00,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:25:00,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742075_1251 (size=12301) 2024-11-16T10:25:00,211 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/fe679ab2f6c846abb5ddbe0067e2e81e 2024-11-16T10:25:00,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/11d7fd850f724e11baecf511a3db3d34 is 50, key is test_row_0/B:col10/1731752699566/Put/seqid=0 2024-11-16T10:25:00,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742076_1252 (size=12301) 2024-11-16T10:25:00,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:00,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752760260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:00,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:00,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:00,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T10:25:00,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:00,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752760367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:00,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:00,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:00,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:00,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:00,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34482 deadline: 1731752760406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:00,407 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_1' in d41ecfe9dd6b68876325d42387a8c316 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:25:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:00,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752760576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:00,624 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/11d7fd850f724e11baecf511a3db3d34 2024-11-16T10:25:00,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/7fbefd9344e444df9705254048051f50 is 50, key is test_row_0/C:col10/1731752699566/Put/seqid=0 2024-11-16T10:25:00,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742077_1253 (size=12301) 2024-11-16T10:25:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T10:25:00,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:00,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752760886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:00,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:00,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:00,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:01,035 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/7fbefd9344e444df9705254048051f50 2024-11-16T10:25:01,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/fe679ab2f6c846abb5ddbe0067e2e81e as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/fe679ab2f6c846abb5ddbe0067e2e81e 2024-11-16T10:25:01,044 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/fe679ab2f6c846abb5ddbe0067e2e81e, entries=150, sequenceid=269, filesize=12.0 K 2024-11-16T10:25:01,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/11d7fd850f724e11baecf511a3db3d34 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/11d7fd850f724e11baecf511a3db3d34 2024-11-16T10:25:01,048 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/11d7fd850f724e11baecf511a3db3d34, entries=150, sequenceid=269, filesize=12.0 K 2024-11-16T10:25:01,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/7fbefd9344e444df9705254048051f50 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/7fbefd9344e444df9705254048051f50 2024-11-16T10:25:01,052 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/7fbefd9344e444df9705254048051f50, entries=150, sequenceid=269, filesize=12.0 K 2024-11-16T10:25:01,053 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for d41ecfe9dd6b68876325d42387a8c316 in 851ms, sequenceid=269, compaction requested=true 2024-11-16T10:25:01,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:25:01,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:01,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-16T10:25:01,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-16T10:25:01,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-16T10:25:01,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0060 sec 2024-11-16T10:25:01,058 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.0100 sec 2024-11-16T10:25:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T10:25:01,186 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:25:01,186 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:25:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-16T10:25:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T10:25:01,188 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:25:01,188 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:25:01,188 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:25:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T10:25:01,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T10:25:01,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:01,340 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing d41ecfe9dd6b68876325d42387a8c316 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-16T10:25:01,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/e4dc94900737461bb9bb3ece8021dfd3 is 50, key is test_row_0/A:col10/1731752700246/Put/seqid=0 2024-11-16T10:25:01,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742078_1254 (size=12301) 2024-11-16T10:25:01,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:25:01,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:25:01,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752761460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:01,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:01,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:01,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:01,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T10:25:01,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752761566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:01,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:01,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:01,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:01,749 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/e4dc94900737461bb9bb3ece8021dfd3 2024-11-16T10:25:01,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/301dfa7a3afd49cd912ca70e75e31173 is 50, key is test_row_0/B:col10/1731752700246/Put/seqid=0 2024-11-16T10:25:01,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742079_1255 (size=12301) 2024-11-16T10:25:01,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:01,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752761777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:01,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:01,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:01,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:01,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T10:25:02,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752762086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:02,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:02,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:02,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:02,168 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/301dfa7a3afd49cd912ca70e75e31173 2024-11-16T10:25:02,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/7557a20a9f0e4adcabf8e1a0dbf48726 is 50, key is test_row_0/C:col10/1731752700246/Put/seqid=0 2024-11-16T10:25:02,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742080_1256 (size=12301) 2024-11-16T10:25:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T10:25:02,582 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/7557a20a9f0e4adcabf8e1a0dbf48726 2024-11-16T10:25:02,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/e4dc94900737461bb9bb3ece8021dfd3 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/e4dc94900737461bb9bb3ece8021dfd3 2024-11-16T10:25:02,589 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/e4dc94900737461bb9bb3ece8021dfd3, entries=150, sequenceid=285, filesize=12.0 K 2024-11-16T10:25:02,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/301dfa7a3afd49cd912ca70e75e31173 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/301dfa7a3afd49cd912ca70e75e31173 2024-11-16T10:25:02,593 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/301dfa7a3afd49cd912ca70e75e31173, entries=150, sequenceid=285, filesize=12.0 K 2024-11-16T10:25:02,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/7557a20a9f0e4adcabf8e1a0dbf48726 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/7557a20a9f0e4adcabf8e1a0dbf48726 2024-11-16T10:25:02,598 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/7557a20a9f0e4adcabf8e1a0dbf48726, entries=150, sequenceid=285, filesize=12.0 K 2024-11-16T10:25:02,599 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for d41ecfe9dd6b68876325d42387a8c316 in 1259ms, sequenceid=285, compaction requested=true 2024-11-16T10:25:02,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:25:02,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:02,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-16T10:25:02,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-16T10:25:02,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:25:02,600 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:25:02,601 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d41ecfe9dd6b68876325d42387a8c316 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-16T10:25:02,603 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-16T10:25:02,603 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4120 sec 2024-11-16T10:25:02,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/e9627b01db884fb9a59f600c35db9c61 is 50, key is test_row_0/A:col10/1731752701454/Put/seqid=0 2024-11-16T10:25:02,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.4170 sec 2024-11-16T10:25:02,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742081_1257 (size=14741) 2024-11-16T10:25:02,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/e9627b01db884fb9a59f600c35db9c61 2024-11-16T10:25:02,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/8c0ceabb78674c46bb68839a2fc29b16 is 50, key is test_row_0/B:col10/1731752701454/Put/seqid=0 2024-11-16T10:25:02,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742082_1258 (size=12301) 2024-11-16T10:25:02,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:02,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752762662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:02,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:02,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:02,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:02,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:02,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752762776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:02,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:02,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:02,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:02,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:02,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752762986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:02,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:02,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:02,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:03,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/8c0ceabb78674c46bb68839a2fc29b16 2024-11-16T10:25:03,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/bbd63ceaee314a10b98d39d2b3fcfd63 is 50, key is test_row_0/C:col10/1731752701454/Put/seqid=0 2024-11-16T10:25:03,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742083_1259 (size=12301) 2024-11-16T10:25:03,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752763296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:03,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T10:25:03,336 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:25:03,336 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:25:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-16T10:25:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T10:25:03,338 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:25:03,338 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:25:03,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:25:03,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/bbd63ceaee314a10b98d39d2b3fcfd63 2024-11-16T10:25:03,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/e9627b01db884fb9a59f600c35db9c61 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/e9627b01db884fb9a59f600c35db9c61 2024-11-16T10:25:03,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T10:25:03,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/e9627b01db884fb9a59f600c35db9c61, entries=200, sequenceid=306, filesize=14.4 K 2024-11-16T10:25:03,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/8c0ceabb78674c46bb68839a2fc29b16 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/8c0ceabb78674c46bb68839a2fc29b16 2024-11-16T10:25:03,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/8c0ceabb78674c46bb68839a2fc29b16, entries=150, sequenceid=306, filesize=12.0 K 2024-11-16T10:25:03,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/bbd63ceaee314a10b98d39d2b3fcfd63 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/bbd63ceaee314a10b98d39d2b3fcfd63 2024-11-16T10:25:03,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/bbd63ceaee314a10b98d39d2b3fcfd63, entries=150, sequenceid=306, filesize=12.0 K 2024-11-16T10:25:03,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for d41ecfe9dd6b68876325d42387a8c316 in 854ms, sequenceid=306, compaction requested=true 2024-11-16T10:25:03,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:25:03,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d41ecfe9dd6b68876325d42387a8c316:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:25:03,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:03,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d41ecfe9dd6b68876325d42387a8c316:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:25:03,456 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-16T10:25:03,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:03,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d41ecfe9dd6b68876325d42387a8c316:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:25:03,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:03,456 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-16T10:25:03,457 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 78664 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-16T10:25:03,457 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): d41ecfe9dd6b68876325d42387a8c316/A is initiating minor compaction (all files) 2024-11-16T10:25:03,457 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d41ecfe9dd6b68876325d42387a8c316/A in TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:03,458 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/23f506fdf4af403bb57dac4d0f30732a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/295ea4eae6714371a53eb51bd8fc7e0f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/c6e70130c27e4a3b90580223cc06fdff, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/fe679ab2f6c846abb5ddbe0067e2e81e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/e4dc94900737461bb9bb3ece8021dfd3, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/e9627b01db884fb9a59f600c35db9c61] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp, totalSize=76.8 K 2024-11-16T10:25:03,458 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73834 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-16T10:25:03,458 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): d41ecfe9dd6b68876325d42387a8c316/B is initiating minor compaction (all files) 2024-11-16T10:25:03,458 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d41ecfe9dd6b68876325d42387a8c316/B in TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:03,458 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/80d592bfe9a24821ad9f2e545fe129e6, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/abb4e6a18003462c9319b104e8467384, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/0f029ed5fe1a4f45bbd97c8380404cbe, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/11d7fd850f724e11baecf511a3db3d34, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/301dfa7a3afd49cd912ca70e75e31173, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/8c0ceabb78674c46bb68839a2fc29b16] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp, totalSize=72.1 K 2024-11-16T10:25:03,458 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23f506fdf4af403bb57dac4d0f30732a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731752697567 2024-11-16T10:25:03,458 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 295ea4eae6714371a53eb51bd8fc7e0f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731752698257 2024-11-16T10:25:03,459 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 80d592bfe9a24821ad9f2e545fe129e6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731752697567 2024-11-16T10:25:03,459 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6e70130c27e4a3b90580223cc06fdff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731752698340 2024-11-16T10:25:03,459 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting abb4e6a18003462c9319b104e8467384, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731752698257 2024-11-16T10:25:03,459 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe679ab2f6c846abb5ddbe0067e2e81e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731752699534 2024-11-16T10:25:03,459 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f029ed5fe1a4f45bbd97c8380404cbe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731752698340 2024-11-16T10:25:03,459 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4dc94900737461bb9bb3ece8021dfd3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731752700230 2024-11-16T10:25:03,460 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 11d7fd850f724e11baecf511a3db3d34, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731752699534 2024-11-16T10:25:03,460 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 301dfa7a3afd49cd912ca70e75e31173, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731752700230 2024-11-16T10:25:03,460 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9627b01db884fb9a59f600c35db9c61, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731752701445 2024-11-16T10:25:03,460 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c0ceabb78674c46bb68839a2fc29b16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731752701454 2024-11-16T10:25:03,471 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d41ecfe9dd6b68876325d42387a8c316#A#compaction#222 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:03,472 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/c2a9178e02b042d7bcf7c43aef2ea23f is 50, key is test_row_0/A:col10/1731752701454/Put/seqid=0 2024-11-16T10:25:03,476 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d41ecfe9dd6b68876325d42387a8c316#B#compaction#223 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:03,477 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/cccd9ebc1b9b4e0ea60ca4c8029a650f is 50, key is test_row_0/B:col10/1731752701454/Put/seqid=0 2024-11-16T10:25:03,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742084_1260 (size=12983) 2024-11-16T10:25:03,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T10:25:03,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:03,491 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing d41ecfe9dd6b68876325d42387a8c316 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-16T10:25:03,493 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/c2a9178e02b042d7bcf7c43aef2ea23f as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/c2a9178e02b042d7bcf7c43aef2ea23f 2024-11-16T10:25:03,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742085_1261 (size=12983) 2024-11-16T10:25:03,499 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/cccd9ebc1b9b4e0ea60ca4c8029a650f as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/cccd9ebc1b9b4e0ea60ca4c8029a650f 2024-11-16T10:25:03,501 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in d41ecfe9dd6b68876325d42387a8c316/A of d41ecfe9dd6b68876325d42387a8c316 into c2a9178e02b042d7bcf7c43aef2ea23f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:03,501 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:25:03,501 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., storeName=d41ecfe9dd6b68876325d42387a8c316/A, priority=10, startTime=1731752703456; duration=0sec 2024-11-16T10:25:03,501 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:03,501 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d41ecfe9dd6b68876325d42387a8c316:A 2024-11-16T10:25:03,501 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-16T10:25:03,505 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in d41ecfe9dd6b68876325d42387a8c316/B of d41ecfe9dd6b68876325d42387a8c316 into cccd9ebc1b9b4e0ea60ca4c8029a650f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:03,505 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:25:03,506 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., storeName=d41ecfe9dd6b68876325d42387a8c316/B, priority=10, startTime=1731752703456; duration=0sec 2024-11-16T10:25:03,506 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:03,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/2d2d8c9d43e048d5b682e09ab45977eb is 50, key is test_row_0/A:col10/1731752702661/Put/seqid=0 2024-11-16T10:25:03,506 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d41ecfe9dd6b68876325d42387a8c316:B 2024-11-16T10:25:03,506 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73834 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-16T10:25:03,506 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): d41ecfe9dd6b68876325d42387a8c316/C is initiating minor compaction (all files) 2024-11-16T10:25:03,506 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d41ecfe9dd6b68876325d42387a8c316/C in TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:03,506 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/67df6bcd436f464ca15b8eff760cd5e3, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/b176345f0aa74a53a2a6f735849c7975, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/00c3e817ac1a41c0abf16ec3ecb79539, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/7fbefd9344e444df9705254048051f50, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/7557a20a9f0e4adcabf8e1a0dbf48726, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/bbd63ceaee314a10b98d39d2b3fcfd63] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp, totalSize=72.1 K 2024-11-16T10:25:03,507 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67df6bcd436f464ca15b8eff760cd5e3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731752697567 2024-11-16T10:25:03,509 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting b176345f0aa74a53a2a6f735849c7975, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731752698257 2024-11-16T10:25:03,510 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00c3e817ac1a41c0abf16ec3ecb79539, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731752698340 2024-11-16T10:25:03,510 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fbefd9344e444df9705254048051f50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731752699534 2024-11-16T10:25:03,510 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7557a20a9f0e4adcabf8e1a0dbf48726, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731752700230 2024-11-16T10:25:03,510 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbd63ceaee314a10b98d39d2b3fcfd63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731752701454 2024-11-16T10:25:03,525 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d41ecfe9dd6b68876325d42387a8c316#C#compaction#225 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:03,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742086_1262 (size=12301) 2024-11-16T10:25:03,526 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/13971819814a46ba91b3cb1be4741917 is 50, key is test_row_0/C:col10/1731752701454/Put/seqid=0 2024-11-16T10:25:03,526 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/2d2d8c9d43e048d5b682e09ab45977eb 2024-11-16T10:25:03,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/31ab2f95e36f453eab6acf452b612b80 is 50, key is test_row_0/B:col10/1731752702661/Put/seqid=0 2024-11-16T10:25:03,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742087_1263 (size=12983) 2024-11-16T10:25:03,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742088_1264 (size=12301) 2024-11-16T10:25:03,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T10:25:03,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:25:03,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:25:03,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:03,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752763753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:03,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:03,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:03,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:03,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752763754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:03,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752763755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:03,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:03,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:03,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:03,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752763806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:03,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:03,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:03,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:03,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752763866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:03,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752763866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:03,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752763866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:03,946 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/13971819814a46ba91b3cb1be4741917 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/13971819814a46ba91b3cb1be4741917 2024-11-16T10:25:03,950 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in d41ecfe9dd6b68876325d42387a8c316/C of d41ecfe9dd6b68876325d42387a8c316 into 13971819814a46ba91b3cb1be4741917(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:03,950 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:25:03,950 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., storeName=d41ecfe9dd6b68876325d42387a8c316/C, priority=10, startTime=1731752703456; duration=0sec 2024-11-16T10:25:03,950 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:03,950 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d41ecfe9dd6b68876325d42387a8c316:C 2024-11-16T10:25:03,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T10:25:03,968 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/31ab2f95e36f453eab6acf452b612b80 2024-11-16T10:25:03,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/7426f6762a9b4ff989577b7de19a1f49 is 50, key is test_row_0/C:col10/1731752702661/Put/seqid=0 2024-11-16T10:25:03,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742089_1265 (size=12301) 2024-11-16T10:25:04,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:04,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752764076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:04,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:04,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752764076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:04,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:04,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752764076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:04,378 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/7426f6762a9b4ff989577b7de19a1f49 2024-11-16T10:25:04,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/2d2d8c9d43e048d5b682e09ab45977eb as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/2d2d8c9d43e048d5b682e09ab45977eb 2024-11-16T10:25:04,385 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/2d2d8c9d43e048d5b682e09ab45977eb, entries=150, sequenceid=321, filesize=12.0 K 2024-11-16T10:25:04,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/31ab2f95e36f453eab6acf452b612b80 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/31ab2f95e36f453eab6acf452b612b80 2024-11-16T10:25:04,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:04,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:04,390 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/31ab2f95e36f453eab6acf452b612b80, entries=150, sequenceid=321, filesize=12.0 K 2024-11-16T10:25:04,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752764386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:04,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752764387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:04,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:04,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752764387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:04,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:04,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:04,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:04,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/7426f6762a9b4ff989577b7de19a1f49 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/7426f6762a9b4ff989577b7de19a1f49 2024-11-16T10:25:04,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:04,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:04,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:04,394 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/7426f6762a9b4ff989577b7de19a1f49, entries=150, sequenceid=321, filesize=12.0 K 2024-11-16T10:25:04,395 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for d41ecfe9dd6b68876325d42387a8c316 in 904ms, sequenceid=321, compaction requested=false 2024-11-16T10:25:04,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:25:04,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:04,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-16T10:25:04,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-16T10:25:04,398 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-16T10:25:04,398 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0580 sec 2024-11-16T10:25:04,399 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.0620 sec 2024-11-16T10:25:04,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T10:25:04,477 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:25:04,477 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:25:04,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-16T10:25:04,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T10:25:04,479 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:25:04,479 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:25:04,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:25:04,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T10:25:04,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T10:25:04,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:04,631 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing d41ecfe9dd6b68876325d42387a8c316 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-16T10:25:04,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/58d57136f33f4888ba196004feb55a4c is 50, key is test_row_0/A:col10/1731752703751/Put/seqid=0 2024-11-16T10:25:04,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742090_1266 (size=12301) 2024-11-16T10:25:04,639 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/58d57136f33f4888ba196004feb55a4c 2024-11-16T10:25:04,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/5de8321f9d344bbe80e76d07512ebed1 is 50, key is test_row_0/B:col10/1731752703751/Put/seqid=0 2024-11-16T10:25:04,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742091_1267 (size=12301) 2024-11-16T10:25:04,650 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/5de8321f9d344bbe80e76d07512ebed1 2024-11-16T10:25:04,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/4929f5bde85c4a749f379d479273e44f is 50, key is test_row_0/C:col10/1731752703751/Put/seqid=0 2024-11-16T10:25:04,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742092_1268 (size=12301) 2024-11-16T10:25:04,685 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/4929f5bde85c4a749f379d479273e44f 2024-11-16T10:25:04,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/58d57136f33f4888ba196004feb55a4c as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/58d57136f33f4888ba196004feb55a4c 2024-11-16T10:25:04,692 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/58d57136f33f4888ba196004feb55a4c, entries=150, sequenceid=345, filesize=12.0 K 2024-11-16T10:25:04,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/5de8321f9d344bbe80e76d07512ebed1 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/5de8321f9d344bbe80e76d07512ebed1 2024-11-16T10:25:04,697 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/5de8321f9d344bbe80e76d07512ebed1, entries=150, sequenceid=345, filesize=12.0 K 2024-11-16T10:25:04,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/4929f5bde85c4a749f379d479273e44f as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/4929f5bde85c4a749f379d479273e44f 2024-11-16T10:25:04,702 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/4929f5bde85c4a749f379d479273e44f, entries=150, sequenceid=345, filesize=12.0 K 2024-11-16T10:25:04,703 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=0 B/0 for d41ecfe9dd6b68876325d42387a8c316 in 72ms, sequenceid=345, compaction requested=true 2024-11-16T10:25:04,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:25:04,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:04,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-16T10:25:04,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-16T10:25:04,708 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-16T10:25:04,708 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 226 msec 2024-11-16T10:25:04,709 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 231 msec 2024-11-16T10:25:04,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T10:25:04,796 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:25:04,796 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:25:04,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-16T10:25:04,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T10:25:04,798 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:25:04,799 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:25:04,799 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:25:04,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:25:04,835 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:25:04,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d41ecfe9dd6b68876325d42387a8c316 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T10:25:04,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/fadf4aed86a64a40b8191ebbe4b57b12 is 50, key is test_row_0/A:col10/1731752704834/Put/seqid=0 2024-11-16T10:25:04,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742093_1269 (size=14741) 2024-11-16T10:25:04,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T10:25:04,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:04,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752764905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:04,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:04,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:04,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:04,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:04,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752764905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:04,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:04,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:04,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:04,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:04,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752764906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:04,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:04,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752764910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:04,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:04,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:04,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:04,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T10:25:04,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:04,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:25:04,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:04,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:04,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:04,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:05,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:05,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752765016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:05,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:05,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752765016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:05,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:05,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752765026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:05,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:05,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:05,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:05,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T10:25:05,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:05,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:25:05,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:05,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:05,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:05,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:05,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T10:25:05,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:05,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752765226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:05,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:05,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:05,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752765226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:05,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:05,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:05,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:05,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:05,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:05,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:05,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752765236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:05,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:05,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:05,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:05,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/fadf4aed86a64a40b8191ebbe4b57b12 2024-11-16T10:25:05,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/d85dd09a5d3a4b3d841f6db5934dcc6e is 50, key is test_row_0/B:col10/1731752704834/Put/seqid=0 2024-11-16T10:25:05,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T10:25:05,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:05,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:25:05,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:05,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:05,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:05,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:05,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742094_1270 (size=12301) 2024-11-16T10:25:05,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T10:25:05,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:05,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:25:05,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:05,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:05,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:05,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:05,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T10:25:05,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:05,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752765536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:05,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:05,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752765536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:05,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:05,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752765546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:05,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:05,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:05,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:05,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T10:25:05,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:05,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:25:05,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:05,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:05,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:05,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/d85dd09a5d3a4b3d841f6db5934dcc6e 2024-11-16T10:25:05,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/24dbf127abe6486d8e953cedbfe57e6d is 50, key is test_row_0/C:col10/1731752704834/Put/seqid=0 2024-11-16T10:25:05,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742095_1271 (size=12301) 2024-11-16T10:25:05,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T10:25:05,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:05,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:25:05,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:05,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:05,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:05,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T10:25:05,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:05,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:25:05,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:05,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:05,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:05,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:05,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:05,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752765926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:05,930 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_0' in d41ecfe9dd6b68876325d42387a8c316 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:25:05,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:05,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:05,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T10:25:06,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T10:25:06,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:06,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:25:06,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:06,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:06,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:06,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752766046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:06,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752766046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:06,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752766056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:06,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:06,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:06,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:06,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/24dbf127abe6486d8e953cedbfe57e6d 2024-11-16T10:25:06,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/fadf4aed86a64a40b8191ebbe4b57b12 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/fadf4aed86a64a40b8191ebbe4b57b12 2024-11-16T10:25:06,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/fadf4aed86a64a40b8191ebbe4b57b12, entries=200, sequenceid=356, filesize=14.4 K 2024-11-16T10:25:06,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/d85dd09a5d3a4b3d841f6db5934dcc6e as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/d85dd09a5d3a4b3d841f6db5934dcc6e 2024-11-16T10:25:06,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/d85dd09a5d3a4b3d841f6db5934dcc6e, entries=150, sequenceid=356, filesize=12.0 K 2024-11-16T10:25:06,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/24dbf127abe6486d8e953cedbfe57e6d as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/24dbf127abe6486d8e953cedbfe57e6d 2024-11-16T10:25:06,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/24dbf127abe6486d8e953cedbfe57e6d, entries=150, sequenceid=356, filesize=12.0 K 2024-11-16T10:25:06,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for d41ecfe9dd6b68876325d42387a8c316 in 1279ms, sequenceid=356, compaction requested=true 2024-11-16T10:25:06,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:25:06,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d41ecfe9dd6b68876325d42387a8c316:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:25:06,114 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:25:06,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:06,114 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:25:06,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d41ecfe9dd6b68876325d42387a8c316:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:25:06,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:06,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d41ecfe9dd6b68876325d42387a8c316:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:25:06,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:06,115 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:25:06,116 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): d41ecfe9dd6b68876325d42387a8c316/B is initiating minor compaction (all files) 2024-11-16T10:25:06,116 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d41ecfe9dd6b68876325d42387a8c316/B in TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:06,116 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/cccd9ebc1b9b4e0ea60ca4c8029a650f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/31ab2f95e36f453eab6acf452b612b80, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/5de8321f9d344bbe80e76d07512ebed1, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/d85dd09a5d3a4b3d841f6db5934dcc6e] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp, totalSize=48.7 K 2024-11-16T10:25:06,116 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52326 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:25:06,116 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): d41ecfe9dd6b68876325d42387a8c316/A is initiating minor compaction (all files) 2024-11-16T10:25:06,116 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d41ecfe9dd6b68876325d42387a8c316/A in TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:06,116 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/c2a9178e02b042d7bcf7c43aef2ea23f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/2d2d8c9d43e048d5b682e09ab45977eb, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/58d57136f33f4888ba196004feb55a4c, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/fadf4aed86a64a40b8191ebbe4b57b12] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp, totalSize=51.1 K 2024-11-16T10:25:06,117 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting cccd9ebc1b9b4e0ea60ca4c8029a650f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731752701454 2024-11-16T10:25:06,117 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2a9178e02b042d7bcf7c43aef2ea23f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731752701454 2024-11-16T10:25:06,117 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d2d8c9d43e048d5b682e09ab45977eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731752702609 2024-11-16T10:25:06,117 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 31ab2f95e36f453eab6acf452b612b80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731752702609 2024-11-16T10:25:06,118 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58d57136f33f4888ba196004feb55a4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731752703751 2024-11-16T10:25:06,118 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 5de8321f9d344bbe80e76d07512ebed1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731752703751 2024-11-16T10:25:06,118 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting fadf4aed86a64a40b8191ebbe4b57b12, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731752704823 2024-11-16T10:25:06,118 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting d85dd09a5d3a4b3d841f6db5934dcc6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731752704823 2024-11-16T10:25:06,129 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d41ecfe9dd6b68876325d42387a8c316#A#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:06,129 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/e31c9c4cc579473fbb1d535b83c6e2a8 is 50, key is test_row_0/A:col10/1731752704834/Put/seqid=0 2024-11-16T10:25:06,133 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d41ecfe9dd6b68876325d42387a8c316#B#compaction#235 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:06,133 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/e866f2a390ef4799b2e65d61d703e1f1 is 50, key is test_row_0/B:col10/1731752704834/Put/seqid=0 2024-11-16T10:25:06,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742096_1272 (size=13119) 2024-11-16T10:25:06,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742097_1273 (size=13119) 2024-11-16T10:25:06,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T10:25:06,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:06,174 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing d41ecfe9dd6b68876325d42387a8c316 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T10:25:06,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/e535da3054fa4f3bbaa14fceac860aa6 is 50, key is test_row_0/A:col10/1731752704898/Put/seqid=0 2024-11-16T10:25:06,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742098_1274 (size=12301) 2024-11-16T10:25:06,565 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/e31c9c4cc579473fbb1d535b83c6e2a8 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/e31c9c4cc579473fbb1d535b83c6e2a8 2024-11-16T10:25:06,571 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/e866f2a390ef4799b2e65d61d703e1f1 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/e866f2a390ef4799b2e65d61d703e1f1 2024-11-16T10:25:06,574 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d41ecfe9dd6b68876325d42387a8c316/A of d41ecfe9dd6b68876325d42387a8c316 into e31c9c4cc579473fbb1d535b83c6e2a8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:06,574 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:25:06,574 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., storeName=d41ecfe9dd6b68876325d42387a8c316/A, priority=12, startTime=1731752706114; duration=0sec 2024-11-16T10:25:06,574 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:06,574 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d41ecfe9dd6b68876325d42387a8c316:A 2024-11-16T10:25:06,574 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:25:06,576 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:25:06,577 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): d41ecfe9dd6b68876325d42387a8c316/C is initiating minor compaction (all files) 2024-11-16T10:25:06,577 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d41ecfe9dd6b68876325d42387a8c316/C in TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:06,577 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/13971819814a46ba91b3cb1be4741917, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/7426f6762a9b4ff989577b7de19a1f49, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/4929f5bde85c4a749f379d479273e44f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/24dbf127abe6486d8e953cedbfe57e6d] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp, totalSize=48.7 K 2024-11-16T10:25:06,578 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13971819814a46ba91b3cb1be4741917, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731752701454 2024-11-16T10:25:06,579 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7426f6762a9b4ff989577b7de19a1f49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731752702609 2024-11-16T10:25:06,580 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d41ecfe9dd6b68876325d42387a8c316/B of d41ecfe9dd6b68876325d42387a8c316 into e866f2a390ef4799b2e65d61d703e1f1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:06,580 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:25:06,580 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., storeName=d41ecfe9dd6b68876325d42387a8c316/B, priority=12, startTime=1731752706114; duration=0sec 2024-11-16T10:25:06,580 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:06,580 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d41ecfe9dd6b68876325d42387a8c316:B 2024-11-16T10:25:06,581 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4929f5bde85c4a749f379d479273e44f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731752703751 2024-11-16T10:25:06,581 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24dbf127abe6486d8e953cedbfe57e6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731752704823 2024-11-16T10:25:06,593 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d41ecfe9dd6b68876325d42387a8c316#C#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:06,594 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/e71cc1bd31a441a9a6792cd0082100f8 is 50, key is test_row_0/C:col10/1731752704834/Put/seqid=0 2024-11-16T10:25:06,602 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/e535da3054fa4f3bbaa14fceac860aa6 2024-11-16T10:25:06,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/8cc4c46ba9f94f27a69250ee7cd5cab7 is 50, key is test_row_0/B:col10/1731752704898/Put/seqid=0 2024-11-16T10:25:06,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742099_1275 (size=13119) 2024-11-16T10:25:06,645 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/e71cc1bd31a441a9a6792cd0082100f8 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/e71cc1bd31a441a9a6792cd0082100f8 2024-11-16T10:25:06,654 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d41ecfe9dd6b68876325d42387a8c316/C of d41ecfe9dd6b68876325d42387a8c316 into e71cc1bd31a441a9a6792cd0082100f8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:06,654 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:25:06,655 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., storeName=d41ecfe9dd6b68876325d42387a8c316/C, priority=12, startTime=1731752706115; duration=0sec 2024-11-16T10:25:06,655 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:06,655 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d41ecfe9dd6b68876325d42387a8c316:C 2024-11-16T10:25:06,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742100_1276 (size=12301) 2024-11-16T10:25:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T10:25:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:25:07,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. as already flushing 2024-11-16T10:25:07,059 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/8cc4c46ba9f94f27a69250ee7cd5cab7 2024-11-16T10:25:07,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/99b74d1986824c73b7b51d9f7ee881a9 is 50, key is test_row_0/C:col10/1731752704898/Put/seqid=0 2024-11-16T10:25:07,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752767075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:07,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34462 deadline: 1731752767076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:07,079 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_0' in d41ecfe9dd6b68876325d42387a8c316 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:25:07,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:07,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:07,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:07,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752767077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:07,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:07,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:07,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:07,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742101_1277 (size=12301) 2024-11-16T10:25:07,089 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/99b74d1986824c73b7b51d9f7ee881a9 2024-11-16T10:25:07,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/e535da3054fa4f3bbaa14fceac860aa6 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/e535da3054fa4f3bbaa14fceac860aa6 2024-11-16T10:25:07,096 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/e535da3054fa4f3bbaa14fceac860aa6, entries=150, sequenceid=381, filesize=12.0 K 2024-11-16T10:25:07,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/8cc4c46ba9f94f27a69250ee7cd5cab7 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/8cc4c46ba9f94f27a69250ee7cd5cab7 2024-11-16T10:25:07,101 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/8cc4c46ba9f94f27a69250ee7cd5cab7, entries=150, sequenceid=381, filesize=12.0 K 2024-11-16T10:25:07,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/99b74d1986824c73b7b51d9f7ee881a9 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/99b74d1986824c73b7b51d9f7ee881a9 2024-11-16T10:25:07,106 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/99b74d1986824c73b7b51d9f7ee881a9, entries=150, sequenceid=381, filesize=12.0 K 2024-11-16T10:25:07,107 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for d41ecfe9dd6b68876325d42387a8c316 in 933ms, sequenceid=381, compaction requested=false 2024-11-16T10:25:07,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:25:07,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:07,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-16T10:25:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-16T10:25:07,109 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-16T10:25:07,109 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3090 sec 2024-11-16T10:25:07,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.3140 sec 2024-11-16T10:25:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:25:07,189 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:25:07,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d41ecfe9dd6b68876325d42387a8c316 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T10:25:07,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/00b6d6004bb24daaa46ac1c44571e9ac is 50, key is test_row_0/A:col10/1731752707188/Put/seqid=0 2024-11-16T10:25:07,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742102_1278 (size=12301) 2024-11-16T10:25:07,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752767261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:07,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752767263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:07,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:07,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:07,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:07,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:07,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752767376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:07,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:07,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:07,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:07,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:07,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752767376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:07,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:07,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:07,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:07,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:07,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752767587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:07,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:07,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:07,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752767587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:07,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:07,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:07,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:07,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:07,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:07,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/00b6d6004bb24daaa46ac1c44571e9ac 2024-11-16T10:25:07,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/ccf9611ae0a442afa9de9c0aa0ed1e38 is 50, key is test_row_0/B:col10/1731752707188/Put/seqid=0 2024-11-16T10:25:07,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742103_1279 (size=12301) 2024-11-16T10:25:07,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/ccf9611ae0a442afa9de9c0aa0ed1e38 2024-11-16T10:25:07,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/1768ad386c3949b89b3eb7602f6d0b9e is 50, key is test_row_0/C:col10/1731752707188/Put/seqid=0 2024-11-16T10:25:07,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742104_1280 (size=12301) 2024-11-16T10:25:07,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:07,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34476 deadline: 1731752767896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:07,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:07,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:07,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:07,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:07,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34446 deadline: 1731752767896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:07,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:07,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:07,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:07,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:07,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34496 deadline: 1731752767946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:07,948 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_0' in d41ecfe9dd6b68876325d42387a8c316 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:25:07,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:07,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:07,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:08,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/1768ad386c3949b89b3eb7602f6d0b9e 2024-11-16T10:25:08,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/00b6d6004bb24daaa46ac1c44571e9ac as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/00b6d6004bb24daaa46ac1c44571e9ac 2024-11-16T10:25:08,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/00b6d6004bb24daaa46ac1c44571e9ac, entries=150, sequenceid=397, filesize=12.0 K 2024-11-16T10:25:08,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/ccf9611ae0a442afa9de9c0aa0ed1e38 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/ccf9611ae0a442afa9de9c0aa0ed1e38 2024-11-16T10:25:08,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/ccf9611ae0a442afa9de9c0aa0ed1e38, entries=150, sequenceid=397, filesize=12.0 K 2024-11-16T10:25:08,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/1768ad386c3949b89b3eb7602f6d0b9e as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/1768ad386c3949b89b3eb7602f6d0b9e 2024-11-16T10:25:08,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/1768ad386c3949b89b3eb7602f6d0b9e, entries=150, sequenceid=397, filesize=12.0 K 2024-11-16T10:25:08,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d41ecfe9dd6b68876325d42387a8c316 in 860ms, sequenceid=397, compaction requested=true 2024-11-16T10:25:08,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:25:08,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d41ecfe9dd6b68876325d42387a8c316:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:25:08,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:08,050 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:25:08,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d41ecfe9dd6b68876325d42387a8c316:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:25:08,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:08,050 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:25:08,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d41ecfe9dd6b68876325d42387a8c316:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:25:08,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:08,051 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:25:08,051 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:25:08,051 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): d41ecfe9dd6b68876325d42387a8c316/B is initiating minor compaction (all files) 2024-11-16T10:25:08,051 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): d41ecfe9dd6b68876325d42387a8c316/A is initiating minor compaction (all files) 2024-11-16T10:25:08,051 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d41ecfe9dd6b68876325d42387a8c316/A in TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:08,051 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/e31c9c4cc579473fbb1d535b83c6e2a8, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/e535da3054fa4f3bbaa14fceac860aa6, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/00b6d6004bb24daaa46ac1c44571e9ac] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp, totalSize=36.8 K 2024-11-16T10:25:08,051 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d41ecfe9dd6b68876325d42387a8c316/B in TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:08,051 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/e866f2a390ef4799b2e65d61d703e1f1, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/8cc4c46ba9f94f27a69250ee7cd5cab7, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/ccf9611ae0a442afa9de9c0aa0ed1e38] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp, totalSize=36.8 K 2024-11-16T10:25:08,052 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting e866f2a390ef4799b2e65d61d703e1f1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731752704823 2024-11-16T10:25:08,052 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting e31c9c4cc579473fbb1d535b83c6e2a8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731752704823 2024-11-16T10:25:08,052 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cc4c46ba9f94f27a69250ee7cd5cab7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731752704898 2024-11-16T10:25:08,052 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting e535da3054fa4f3bbaa14fceac860aa6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731752704898 2024-11-16T10:25:08,052 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting ccf9611ae0a442afa9de9c0aa0ed1e38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731752707069 2024-11-16T10:25:08,052 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00b6d6004bb24daaa46ac1c44571e9ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731752707069 2024-11-16T10:25:08,060 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d41ecfe9dd6b68876325d42387a8c316#A#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:08,060 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/fe97b2ea5d4f4701b83dd2843a9bacc8 is 50, key is test_row_0/A:col10/1731752707188/Put/seqid=0 2024-11-16T10:25:08,066 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d41ecfe9dd6b68876325d42387a8c316#B#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:08,067 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/ebdd02ad3c3e42baa9c14a825291defe is 50, key is test_row_0/B:col10/1731752707188/Put/seqid=0 2024-11-16T10:25:08,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742105_1281 (size=13221) 2024-11-16T10:25:08,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742106_1282 (size=13221) 2024-11-16T10:25:08,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:25:08,408 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:25:08,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d41ecfe9dd6b68876325d42387a8c316 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T10:25:08,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/477589415e2446789952b5ddf9578699 is 50, key is test_row_0/A:col10/1731752708407/Put/seqid=0 2024-11-16T10:25:08,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742107_1283 (size=14741) 2024-11-16T10:25:08,423 INFO [Thread-957 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-957. 2024-11-16T10:25:08,423 DEBUG [Thread-957 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:25:08,423 DEBUG [Thread-957 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:08,423 DEBUG [Thread-957 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:08,423 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:25:08,425 INFO [Thread-970 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-970. 2024-11-16T10:25:08,425 DEBUG [Thread-970 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:25:08,425 DEBUG [Thread-970 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:08,425 DEBUG [Thread-970 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:08,425 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:25:08,426 INFO [Thread-961 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-961. 2024-11-16T10:25:08,426 DEBUG [Thread-961 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:25:08,426 DEBUG [Thread-961 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:08,426 DEBUG [Thread-961 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:08,426 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:25:08,428 INFO [Thread-972 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-972. 2024-11-16T10:25:08,428 INFO [Thread-964 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-964. 2024-11-16T10:25:08,428 DEBUG [Thread-972 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:25:08,428 DEBUG [Thread-972 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:08,428 DEBUG [Thread-964 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:25:08,428 DEBUG [Thread-964 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:08,428 DEBUG [Thread-972 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:08,428 DEBUG [Thread-964 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:08,428 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:25:08,428 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:25:08,429 INFO [Thread-968 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-968. 2024-11-16T10:25:08,429 DEBUG [Thread-968 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:25:08,429 DEBUG [Thread-968 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:08,429 DEBUG [Thread-968 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:08,429 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:25:08,430 INFO [Thread-966 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-966. 2024-11-16T10:25:08,430 DEBUG [Thread-966 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:25:08,430 DEBUG [Thread-966 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:08,430 DEBUG [Thread-966 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:08,430 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:25:08,481 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/fe97b2ea5d4f4701b83dd2843a9bacc8 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/fe97b2ea5d4f4701b83dd2843a9bacc8 2024-11-16T10:25:08,489 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d41ecfe9dd6b68876325d42387a8c316/A of d41ecfe9dd6b68876325d42387a8c316 into fe97b2ea5d4f4701b83dd2843a9bacc8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:08,489 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:25:08,490 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., storeName=d41ecfe9dd6b68876325d42387a8c316/A, priority=13, startTime=1731752708050; duration=0sec 2024-11-16T10:25:08,490 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:08,490 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d41ecfe9dd6b68876325d42387a8c316:A 2024-11-16T10:25:08,490 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:25:08,491 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:25:08,491 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): d41ecfe9dd6b68876325d42387a8c316/C is initiating minor compaction (all files) 2024-11-16T10:25:08,491 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d41ecfe9dd6b68876325d42387a8c316/C in TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:08,491 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/e71cc1bd31a441a9a6792cd0082100f8, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/99b74d1986824c73b7b51d9f7ee881a9, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/1768ad386c3949b89b3eb7602f6d0b9e] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp, totalSize=36.8 K 2024-11-16T10:25:08,492 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting e71cc1bd31a441a9a6792cd0082100f8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731752704823 2024-11-16T10:25:08,492 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/ebdd02ad3c3e42baa9c14a825291defe as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/ebdd02ad3c3e42baa9c14a825291defe 2024-11-16T10:25:08,492 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99b74d1986824c73b7b51d9f7ee881a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731752704898 2024-11-16T10:25:08,493 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1768ad386c3949b89b3eb7602f6d0b9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731752707069 2024-11-16T10:25:08,498 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d41ecfe9dd6b68876325d42387a8c316/B of d41ecfe9dd6b68876325d42387a8c316 into ebdd02ad3c3e42baa9c14a825291defe(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:08,498 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:25:08,498 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., storeName=d41ecfe9dd6b68876325d42387a8c316/B, priority=13, startTime=1731752708050; duration=0sec 2024-11-16T10:25:08,498 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:08,498 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d41ecfe9dd6b68876325d42387a8c316:B 2024-11-16T10:25:08,501 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d41ecfe9dd6b68876325d42387a8c316#C#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:08,502 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/e84fb3fc45764573b4348e1fba59e9d8 is 50, key is test_row_0/C:col10/1731752707188/Put/seqid=0 2024-11-16T10:25:08,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742108_1284 (size=13221) 2024-11-16T10:25:08,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/477589415e2446789952b5ddf9578699 2024-11-16T10:25:08,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/2c2de7dc808a405ea494a4fcb6e4ec85 is 50, key is test_row_0/B:col10/1731752708407/Put/seqid=0 2024-11-16T10:25:08,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742109_1285 (size=12301) 2024-11-16T10:25:08,916 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/e84fb3fc45764573b4348e1fba59e9d8 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/e84fb3fc45764573b4348e1fba59e9d8 2024-11-16T10:25:08,923 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d41ecfe9dd6b68876325d42387a8c316/C of d41ecfe9dd6b68876325d42387a8c316 into e84fb3fc45764573b4348e1fba59e9d8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:08,923 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:25:08,923 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., storeName=d41ecfe9dd6b68876325d42387a8c316/C, priority=13, startTime=1731752708050; duration=0sec 2024-11-16T10:25:08,923 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:08,923 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d41ecfe9dd6b68876325d42387a8c316:C 2024-11-16T10:25:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T10:25:08,957 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:25:09,091 INFO [Thread-953 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-953. 2024-11-16T10:25:09,091 DEBUG [Thread-953 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:25:09,091 DEBUG [Thread-953 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:09,091 DEBUG [Thread-953 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:09,091 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:25:09,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/2c2de7dc808a405ea494a4fcb6e4ec85 2024-11-16T10:25:09,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/5a78b3dd511d41ca86ba7080581e4ae7 is 50, key is test_row_0/C:col10/1731752708407/Put/seqid=0 2024-11-16T10:25:09,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742110_1286 (size=12301) 2024-11-16T10:25:09,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/5a78b3dd511d41ca86ba7080581e4ae7 2024-11-16T10:25:09,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/477589415e2446789952b5ddf9578699 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/477589415e2446789952b5ddf9578699 2024-11-16T10:25:09,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/477589415e2446789952b5ddf9578699, entries=200, sequenceid=422, filesize=14.4 K 2024-11-16T10:25:09,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/2c2de7dc808a405ea494a4fcb6e4ec85 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/2c2de7dc808a405ea494a4fcb6e4ec85 2024-11-16T10:25:09,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/2c2de7dc808a405ea494a4fcb6e4ec85, entries=150, sequenceid=422, filesize=12.0 K 2024-11-16T10:25:09,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/5a78b3dd511d41ca86ba7080581e4ae7 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/5a78b3dd511d41ca86ba7080581e4ae7 2024-11-16T10:25:09,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/5a78b3dd511d41ca86ba7080581e4ae7, entries=150, sequenceid=422, filesize=12.0 K 2024-11-16T10:25:09,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for d41ecfe9dd6b68876325d42387a8c316 in 1265ms, sequenceid=422, compaction requested=false 2024-11-16T10:25:09,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:25:10,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:25:10,480 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:25:10,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d41ecfe9dd6b68876325d42387a8c316 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T10:25:10,480 INFO [Thread-959 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-959. 2024-11-16T10:25:10,481 DEBUG [Thread-959 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:25:10,481 DEBUG [Thread-959 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:10,481 DEBUG [Thread-959 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:10,481 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:25:10,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/df45b39f62034e75ba25944790be9b9c is 50, key is test_row_0/A:col10/1731752709087/Put/seqid=0 2024-11-16T10:25:10,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742111_1287 (size=12301) 2024-11-16T10:25:10,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/df45b39f62034e75ba25944790be9b9c 2024-11-16T10:25:10,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/2553740bfc454c8eaa9e9411c2a50fbd is 50, key is test_row_0/B:col10/1731752709087/Put/seqid=0 2024-11-16T10:25:10,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742112_1288 (size=12301) 2024-11-16T10:25:11,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/2553740bfc454c8eaa9e9411c2a50fbd 2024-11-16T10:25:11,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/eb547d893e7643d18633ed3b262a613a is 50, key is test_row_0/C:col10/1731752709087/Put/seqid=0 2024-11-16T10:25:11,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742113_1289 (size=12301) 2024-11-16T10:25:11,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/eb547d893e7643d18633ed3b262a613a 2024-11-16T10:25:11,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/df45b39f62034e75ba25944790be9b9c as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/df45b39f62034e75ba25944790be9b9c 2024-11-16T10:25:11,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/df45b39f62034e75ba25944790be9b9c, entries=150, sequenceid=437, filesize=12.0 K 2024-11-16T10:25:11,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/2553740bfc454c8eaa9e9411c2a50fbd as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/2553740bfc454c8eaa9e9411c2a50fbd 2024-11-16T10:25:11,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/2553740bfc454c8eaa9e9411c2a50fbd, entries=150, sequenceid=437, filesize=12.0 K 2024-11-16T10:25:11,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/eb547d893e7643d18633ed3b262a613a as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/eb547d893e7643d18633ed3b262a613a 2024-11-16T10:25:11,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/eb547d893e7643d18633ed3b262a613a, entries=150, sequenceid=437, filesize=12.0 K 2024-11-16T10:25:11,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for d41ecfe9dd6b68876325d42387a8c316 in 1263ms, sequenceid=437, compaction requested=true 2024-11-16T10:25:11,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:25:11,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d41ecfe9dd6b68876325d42387a8c316:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:25:11,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:11,743 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:25:11,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d41ecfe9dd6b68876325d42387a8c316:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:25:11,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:11,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d41ecfe9dd6b68876325d42387a8c316:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:25:11,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:11,743 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:25:11,744 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:25:11,744 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:25:11,744 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): d41ecfe9dd6b68876325d42387a8c316/B is initiating minor compaction (all files) 2024-11-16T10:25:11,744 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): d41ecfe9dd6b68876325d42387a8c316/A is initiating minor compaction (all files) 2024-11-16T10:25:11,744 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d41ecfe9dd6b68876325d42387a8c316/B in TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:11,744 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d41ecfe9dd6b68876325d42387a8c316/A in TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:11,744 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/fe97b2ea5d4f4701b83dd2843a9bacc8, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/477589415e2446789952b5ddf9578699, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/df45b39f62034e75ba25944790be9b9c] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp, totalSize=39.3 K 2024-11-16T10:25:11,744 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/ebdd02ad3c3e42baa9c14a825291defe, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/2c2de7dc808a405ea494a4fcb6e4ec85, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/2553740bfc454c8eaa9e9411c2a50fbd] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp, totalSize=36.9 K 2024-11-16T10:25:11,744 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting ebdd02ad3c3e42baa9c14a825291defe, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731752707069 2024-11-16T10:25:11,744 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe97b2ea5d4f4701b83dd2843a9bacc8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731752707069 2024-11-16T10:25:11,745 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c2de7dc808a405ea494a4fcb6e4ec85, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731752707262 2024-11-16T10:25:11,745 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 477589415e2446789952b5ddf9578699, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731752707259 2024-11-16T10:25:11,745 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 2553740bfc454c8eaa9e9411c2a50fbd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1731752708416 2024-11-16T10:25:11,745 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting df45b39f62034e75ba25944790be9b9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1731752708416 2024-11-16T10:25:11,752 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d41ecfe9dd6b68876325d42387a8c316#B#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:11,752 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d41ecfe9dd6b68876325d42387a8c316#A#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:11,752 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/e9207c1cc9b943ce9a2ef672bb10ae75 is 50, key is test_row_0/B:col10/1731752709087/Put/seqid=0 2024-11-16T10:25:11,752 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/8156c0a74ef7486db97998d52f423a3b is 50, key is test_row_0/A:col10/1731752709087/Put/seqid=0 2024-11-16T10:25:11,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742115_1291 (size=13323) 2024-11-16T10:25:11,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742114_1290 (size=13323) 2024-11-16T10:25:11,989 INFO [Thread-955 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-955. 2024-11-16T10:25:11,989 DEBUG [Thread-955 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:25:11,989 DEBUG [Thread-955 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:11,989 DEBUG [Thread-955 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:11,989 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:25:11,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T10:25:11,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 183 2024-11-16T10:25:11,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-16T10:25:11,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-16T10:25:11,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-16T10:25:11,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-16T10:25:11,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T10:25:11,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3417 2024-11-16T10:25:11,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3356 2024-11-16T10:25:11,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3367 2024-11-16T10:25:11,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3390 2024-11-16T10:25:11,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3361 2024-11-16T10:25:11,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T10:25:11,990 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T10:25:11,990 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T10:25:11,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:11,991 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:11,991 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:25:11,992 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T10:25:11,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T10:25:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T10:25:11,997 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752711997"}]},"ts":"1731752711997"} 2024-11-16T10:25:11,999 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T10:25:11,999 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T10:25:12,000 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T10:25:12,002 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d41ecfe9dd6b68876325d42387a8c316, UNASSIGN}] 2024-11-16T10:25:12,003 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d41ecfe9dd6b68876325d42387a8c316, UNASSIGN 2024-11-16T10:25:12,004 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=d41ecfe9dd6b68876325d42387a8c316, regionState=CLOSING, regionLocation=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:12,005 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d41ecfe9dd6b68876325d42387a8c316, UNASSIGN because future has completed 2024-11-16T10:25:12,006 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T10:25:12,006 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506}] 2024-11-16T10:25:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T10:25:12,160 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:25:12,160 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T10:25:12,160 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing d41ecfe9dd6b68876325d42387a8c316, disabling compactions & flushes 2024-11-16T10:25:12,161 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:12,170 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/e9207c1cc9b943ce9a2ef672bb10ae75 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/e9207c1cc9b943ce9a2ef672bb10ae75 2024-11-16T10:25:12,170 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/8156c0a74ef7486db97998d52f423a3b as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/8156c0a74ef7486db97998d52f423a3b 2024-11-16T10:25:12,174 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d41ecfe9dd6b68876325d42387a8c316/B of d41ecfe9dd6b68876325d42387a8c316 into e9207c1cc9b943ce9a2ef672bb10ae75(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:12,174 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d41ecfe9dd6b68876325d42387a8c316/A of d41ecfe9dd6b68876325d42387a8c316 into 8156c0a74ef7486db97998d52f423a3b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:12,174 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:25:12,174 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d41ecfe9dd6b68876325d42387a8c316: 2024-11-16T10:25:12,174 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., storeName=d41ecfe9dd6b68876325d42387a8c316/A, priority=13, startTime=1731752711743; duration=0sec 2024-11-16T10:25:12,174 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., storeName=d41ecfe9dd6b68876325d42387a8c316/B, priority=13, startTime=1731752711743; duration=0sec 2024-11-16T10:25:12,174 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:12,174 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:12,174 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:12,175 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:12,175 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d41ecfe9dd6b68876325d42387a8c316:A 2024-11-16T10:25:12,175 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. after waiting 0 ms 2024-11-16T10:25:12,175 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d41ecfe9dd6b68876325d42387a8c316:B 2024-11-16T10:25:12,175 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:12,175 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing d41ecfe9dd6b68876325d42387a8c316 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-16T10:25:12,175 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. because compaction request was cancelled 2024-11-16T10:25:12,175 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d41ecfe9dd6b68876325d42387a8c316:C 2024-11-16T10:25:12,178 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/6bab9b3ad01346d49059ca8622d281de is 50, key is test_row_0/A:col10/1731752711987/Put/seqid=0 2024-11-16T10:25:12,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742116_1292 (size=7415) 2024-11-16T10:25:12,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T10:25:12,584 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/6bab9b3ad01346d49059ca8622d281de 2024-11-16T10:25:12,599 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/3c828667a1c542ef99098c1db14eedc5 is 50, key is test_row_0/B:col10/1731752711987/Put/seqid=0 2024-11-16T10:25:12,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742117_1293 (size=7415) 2024-11-16T10:25:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T10:25:13,005 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/3c828667a1c542ef99098c1db14eedc5 2024-11-16T10:25:13,017 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/40afe031c23b4ede883e5b52a465d4d6 is 50, key is test_row_0/C:col10/1731752711987/Put/seqid=0 2024-11-16T10:25:13,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742118_1294 (size=7415) 2024-11-16T10:25:13,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T10:25:13,423 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/40afe031c23b4ede883e5b52a465d4d6 2024-11-16T10:25:13,435 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/A/6bab9b3ad01346d49059ca8622d281de as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/6bab9b3ad01346d49059ca8622d281de 2024-11-16T10:25:13,441 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/6bab9b3ad01346d49059ca8622d281de, entries=50, sequenceid=443, filesize=7.2 K 2024-11-16T10:25:13,442 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/B/3c828667a1c542ef99098c1db14eedc5 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/3c828667a1c542ef99098c1db14eedc5 2024-11-16T10:25:13,446 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/3c828667a1c542ef99098c1db14eedc5, entries=50, sequenceid=443, filesize=7.2 K 2024-11-16T10:25:13,447 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/.tmp/C/40afe031c23b4ede883e5b52a465d4d6 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/40afe031c23b4ede883e5b52a465d4d6 2024-11-16T10:25:13,452 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/40afe031c23b4ede883e5b52a465d4d6, entries=50, sequenceid=443, filesize=7.2 K 2024-11-16T10:25:13,453 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for d41ecfe9dd6b68876325d42387a8c316 in 1278ms, sequenceid=443, compaction requested=true 2024-11-16T10:25:13,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/079c5f79d974494f962ca2ab87ff4e10, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/46753f832d53430a9e8c5621eadd8ede, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/65a8a78854f340f7b4c6d253633ff9b7, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/0d83833f0cc6409aaef32788d6b008e4, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/2c51c0881613426684a4a9efeb9e3f49, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/4cd7cc0d896444069396ec33c4578bac, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/842f97f962044168aeb97cc499dd56e4, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/d10f0e533acc438f8fe9f6e865c2ea36, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/0c28b96c01a24b4398f669fcd913a715, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/95bd5cd41ae04f26914a6ef4d860c417, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/5fd9b2751d8a466e87dc10e82c2e65f9, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/ce45b4528e6142c58557482caf8f0509, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/824d1832f5cc4cc49fbdf232955689ba, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/176cc05b25be453aa2053e54bb29aa56, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/23f506fdf4af403bb57dac4d0f30732a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/295ea4eae6714371a53eb51bd8fc7e0f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/c6e70130c27e4a3b90580223cc06fdff, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/fe679ab2f6c846abb5ddbe0067e2e81e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/e4dc94900737461bb9bb3ece8021dfd3, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/e9627b01db884fb9a59f600c35db9c61, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/c2a9178e02b042d7bcf7c43aef2ea23f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/2d2d8c9d43e048d5b682e09ab45977eb, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/58d57136f33f4888ba196004feb55a4c, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/fadf4aed86a64a40b8191ebbe4b57b12, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/e31c9c4cc579473fbb1d535b83c6e2a8, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/e535da3054fa4f3bbaa14fceac860aa6, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/fe97b2ea5d4f4701b83dd2843a9bacc8, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/00b6d6004bb24daaa46ac1c44571e9ac, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/477589415e2446789952b5ddf9578699, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/df45b39f62034e75ba25944790be9b9c] to archive 2024-11-16T10:25:13,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:25:13,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/079c5f79d974494f962ca2ab87ff4e10 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/079c5f79d974494f962ca2ab87ff4e10 2024-11-16T10:25:13,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/46753f832d53430a9e8c5621eadd8ede to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/46753f832d53430a9e8c5621eadd8ede 2024-11-16T10:25:13,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/65a8a78854f340f7b4c6d253633ff9b7 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/65a8a78854f340f7b4c6d253633ff9b7 2024-11-16T10:25:13,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/0d83833f0cc6409aaef32788d6b008e4 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/0d83833f0cc6409aaef32788d6b008e4 2024-11-16T10:25:13,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/2c51c0881613426684a4a9efeb9e3f49 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/2c51c0881613426684a4a9efeb9e3f49 2024-11-16T10:25:13,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/4cd7cc0d896444069396ec33c4578bac to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/4cd7cc0d896444069396ec33c4578bac 2024-11-16T10:25:13,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/842f97f962044168aeb97cc499dd56e4 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/842f97f962044168aeb97cc499dd56e4 2024-11-16T10:25:13,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/d10f0e533acc438f8fe9f6e865c2ea36 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/d10f0e533acc438f8fe9f6e865c2ea36 2024-11-16T10:25:13,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/0c28b96c01a24b4398f669fcd913a715 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/0c28b96c01a24b4398f669fcd913a715 2024-11-16T10:25:13,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/95bd5cd41ae04f26914a6ef4d860c417 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/95bd5cd41ae04f26914a6ef4d860c417 2024-11-16T10:25:13,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/5fd9b2751d8a466e87dc10e82c2e65f9 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/5fd9b2751d8a466e87dc10e82c2e65f9 2024-11-16T10:25:13,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/ce45b4528e6142c58557482caf8f0509 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/ce45b4528e6142c58557482caf8f0509 2024-11-16T10:25:13,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/824d1832f5cc4cc49fbdf232955689ba to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/824d1832f5cc4cc49fbdf232955689ba 2024-11-16T10:25:13,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/176cc05b25be453aa2053e54bb29aa56 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/176cc05b25be453aa2053e54bb29aa56 2024-11-16T10:25:13,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/23f506fdf4af403bb57dac4d0f30732a to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/23f506fdf4af403bb57dac4d0f30732a 2024-11-16T10:25:13,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/295ea4eae6714371a53eb51bd8fc7e0f to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/295ea4eae6714371a53eb51bd8fc7e0f 2024-11-16T10:25:13,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/c6e70130c27e4a3b90580223cc06fdff to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/c6e70130c27e4a3b90580223cc06fdff 2024-11-16T10:25:13,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/fe679ab2f6c846abb5ddbe0067e2e81e to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/fe679ab2f6c846abb5ddbe0067e2e81e 2024-11-16T10:25:13,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/e4dc94900737461bb9bb3ece8021dfd3 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/e4dc94900737461bb9bb3ece8021dfd3 2024-11-16T10:25:13,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/e9627b01db884fb9a59f600c35db9c61 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/e9627b01db884fb9a59f600c35db9c61 2024-11-16T10:25:13,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/c2a9178e02b042d7bcf7c43aef2ea23f to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/c2a9178e02b042d7bcf7c43aef2ea23f 2024-11-16T10:25:13,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/2d2d8c9d43e048d5b682e09ab45977eb to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/2d2d8c9d43e048d5b682e09ab45977eb 2024-11-16T10:25:13,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/58d57136f33f4888ba196004feb55a4c to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/58d57136f33f4888ba196004feb55a4c 2024-11-16T10:25:13,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/fadf4aed86a64a40b8191ebbe4b57b12 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/fadf4aed86a64a40b8191ebbe4b57b12 2024-11-16T10:25:13,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/e31c9c4cc579473fbb1d535b83c6e2a8 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/e31c9c4cc579473fbb1d535b83c6e2a8 2024-11-16T10:25:13,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/e535da3054fa4f3bbaa14fceac860aa6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/e535da3054fa4f3bbaa14fceac860aa6 2024-11-16T10:25:13,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/fe97b2ea5d4f4701b83dd2843a9bacc8 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/fe97b2ea5d4f4701b83dd2843a9bacc8 2024-11-16T10:25:13,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/00b6d6004bb24daaa46ac1c44571e9ac to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/00b6d6004bb24daaa46ac1c44571e9ac 2024-11-16T10:25:13,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/477589415e2446789952b5ddf9578699 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/477589415e2446789952b5ddf9578699 2024-11-16T10:25:13,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/df45b39f62034e75ba25944790be9b9c to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/df45b39f62034e75ba25944790be9b9c 2024-11-16T10:25:13,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/5b68f06626db45c9bf2da72332508d3a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/7ff10646cb4f4111afc4d186044b8e34, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/7f64b0b2c30d48d889888270a77a4730, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/d6c3ecae3a7d4d7ea3e482eb579f64d3, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/721baee8f1624cf4ac4f032c20745ebb, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/d1a1b27513ba4568836a2fb1b3964987, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/a935947dbda041fc8f9858252c3fcf11, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/b3ceedbf4dcb421e9f6110acdfef29b8, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/af62138b27c7443996787c342a217f53, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/5b3c26cbf8544a73bcb5a1ddebfd58f8, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/38e42773d43a41ef9cc4d4bb5efe4951, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/bf5f5dcba33e4e30a21c1102581ddfed, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/5edfc535335245179c12b1056c7af95e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/80d592bfe9a24821ad9f2e545fe129e6, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/e77beacbb74245d191a9292381cb941e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/abb4e6a18003462c9319b104e8467384, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/0f029ed5fe1a4f45bbd97c8380404cbe, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/11d7fd850f724e11baecf511a3db3d34, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/301dfa7a3afd49cd912ca70e75e31173, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/cccd9ebc1b9b4e0ea60ca4c8029a650f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/8c0ceabb78674c46bb68839a2fc29b16, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/31ab2f95e36f453eab6acf452b612b80, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/5de8321f9d344bbe80e76d07512ebed1, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/e866f2a390ef4799b2e65d61d703e1f1, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/d85dd09a5d3a4b3d841f6db5934dcc6e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/8cc4c46ba9f94f27a69250ee7cd5cab7, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/ebdd02ad3c3e42baa9c14a825291defe, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/ccf9611ae0a442afa9de9c0aa0ed1e38, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/2c2de7dc808a405ea494a4fcb6e4ec85, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/2553740bfc454c8eaa9e9411c2a50fbd] to archive 2024-11-16T10:25:13,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:25:13,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/5b68f06626db45c9bf2da72332508d3a to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/5b68f06626db45c9bf2da72332508d3a 2024-11-16T10:25:13,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/7ff10646cb4f4111afc4d186044b8e34 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/7ff10646cb4f4111afc4d186044b8e34 2024-11-16T10:25:13,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/7f64b0b2c30d48d889888270a77a4730 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/7f64b0b2c30d48d889888270a77a4730 2024-11-16T10:25:13,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/d6c3ecae3a7d4d7ea3e482eb579f64d3 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/d6c3ecae3a7d4d7ea3e482eb579f64d3 2024-11-16T10:25:13,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/721baee8f1624cf4ac4f032c20745ebb to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/721baee8f1624cf4ac4f032c20745ebb 2024-11-16T10:25:13,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/d1a1b27513ba4568836a2fb1b3964987 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/d1a1b27513ba4568836a2fb1b3964987 2024-11-16T10:25:13,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/a935947dbda041fc8f9858252c3fcf11 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/a935947dbda041fc8f9858252c3fcf11 2024-11-16T10:25:13,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/b3ceedbf4dcb421e9f6110acdfef29b8 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/b3ceedbf4dcb421e9f6110acdfef29b8 2024-11-16T10:25:13,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/af62138b27c7443996787c342a217f53 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/af62138b27c7443996787c342a217f53 2024-11-16T10:25:13,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/5b3c26cbf8544a73bcb5a1ddebfd58f8 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/5b3c26cbf8544a73bcb5a1ddebfd58f8 2024-11-16T10:25:13,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/38e42773d43a41ef9cc4d4bb5efe4951 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/38e42773d43a41ef9cc4d4bb5efe4951 2024-11-16T10:25:13,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/bf5f5dcba33e4e30a21c1102581ddfed to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/bf5f5dcba33e4e30a21c1102581ddfed 2024-11-16T10:25:13,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/5edfc535335245179c12b1056c7af95e to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/5edfc535335245179c12b1056c7af95e 2024-11-16T10:25:13,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/80d592bfe9a24821ad9f2e545fe129e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/80d592bfe9a24821ad9f2e545fe129e6 2024-11-16T10:25:13,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/e77beacbb74245d191a9292381cb941e to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/e77beacbb74245d191a9292381cb941e 2024-11-16T10:25:13,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/abb4e6a18003462c9319b104e8467384 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/abb4e6a18003462c9319b104e8467384 2024-11-16T10:25:13,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/0f029ed5fe1a4f45bbd97c8380404cbe to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/0f029ed5fe1a4f45bbd97c8380404cbe 2024-11-16T10:25:13,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/11d7fd850f724e11baecf511a3db3d34 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/11d7fd850f724e11baecf511a3db3d34 2024-11-16T10:25:13,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/301dfa7a3afd49cd912ca70e75e31173 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/301dfa7a3afd49cd912ca70e75e31173 2024-11-16T10:25:13,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/cccd9ebc1b9b4e0ea60ca4c8029a650f to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/cccd9ebc1b9b4e0ea60ca4c8029a650f 2024-11-16T10:25:13,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/8c0ceabb78674c46bb68839a2fc29b16 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/8c0ceabb78674c46bb68839a2fc29b16 2024-11-16T10:25:13,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/31ab2f95e36f453eab6acf452b612b80 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/31ab2f95e36f453eab6acf452b612b80 2024-11-16T10:25:13,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/5de8321f9d344bbe80e76d07512ebed1 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/5de8321f9d344bbe80e76d07512ebed1 2024-11-16T10:25:13,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/e866f2a390ef4799b2e65d61d703e1f1 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/e866f2a390ef4799b2e65d61d703e1f1 2024-11-16T10:25:13,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/d85dd09a5d3a4b3d841f6db5934dcc6e to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/d85dd09a5d3a4b3d841f6db5934dcc6e 2024-11-16T10:25:13,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/8cc4c46ba9f94f27a69250ee7cd5cab7 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/8cc4c46ba9f94f27a69250ee7cd5cab7 2024-11-16T10:25:13,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/ebdd02ad3c3e42baa9c14a825291defe to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/ebdd02ad3c3e42baa9c14a825291defe 2024-11-16T10:25:13,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/ccf9611ae0a442afa9de9c0aa0ed1e38 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/ccf9611ae0a442afa9de9c0aa0ed1e38 2024-11-16T10:25:13,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/2c2de7dc808a405ea494a4fcb6e4ec85 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/2c2de7dc808a405ea494a4fcb6e4ec85 2024-11-16T10:25:13,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/2553740bfc454c8eaa9e9411c2a50fbd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/2553740bfc454c8eaa9e9411c2a50fbd 2024-11-16T10:25:13,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/eafbc0a753484a38b62bfbab850c5317, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/9615d03983c64ead93a5add93495ae12, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/cb0a202d233b48fd85a3911f0e19191e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/27ff40ddcd3249c7b7d0eaffa6bc3520, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/a7d3c98134794f0abd6f182178564d1c, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/fb8ab969a5e64328958c0e172680696f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/145f63603d414a548e1572ab5132b06f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/a35810f51bbc4ef88da4060895b5ea80, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/8d05420950994a03badd5bdf5be6a450, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/04b6b0d40a4a450593ea3be60dc6749e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/0351adc88b994c80825f2e817c327573, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/40ee0f215787447ea2f5ef6546b6ddef, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/e72b6ccfc85642c6bde751c3f2fc08b1, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/67df6bcd436f464ca15b8eff760cd5e3, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/a9a67a2de22d41b4a7d9c4b6f640c635, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/b176345f0aa74a53a2a6f735849c7975, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/00c3e817ac1a41c0abf16ec3ecb79539, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/7fbefd9344e444df9705254048051f50, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/7557a20a9f0e4adcabf8e1a0dbf48726, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/13971819814a46ba91b3cb1be4741917, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/bbd63ceaee314a10b98d39d2b3fcfd63, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/7426f6762a9b4ff989577b7de19a1f49, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/4929f5bde85c4a749f379d479273e44f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/e71cc1bd31a441a9a6792cd0082100f8, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/24dbf127abe6486d8e953cedbfe57e6d, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/99b74d1986824c73b7b51d9f7ee881a9, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/1768ad386c3949b89b3eb7602f6d0b9e] to archive 2024-11-16T10:25:13,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:25:13,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/eafbc0a753484a38b62bfbab850c5317 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/eafbc0a753484a38b62bfbab850c5317 2024-11-16T10:25:13,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/9615d03983c64ead93a5add93495ae12 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/9615d03983c64ead93a5add93495ae12 2024-11-16T10:25:13,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/cb0a202d233b48fd85a3911f0e19191e to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/cb0a202d233b48fd85a3911f0e19191e 2024-11-16T10:25:13,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/27ff40ddcd3249c7b7d0eaffa6bc3520 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/27ff40ddcd3249c7b7d0eaffa6bc3520 2024-11-16T10:25:13,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/a7d3c98134794f0abd6f182178564d1c to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/a7d3c98134794f0abd6f182178564d1c 2024-11-16T10:25:13,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/fb8ab969a5e64328958c0e172680696f to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/fb8ab969a5e64328958c0e172680696f 2024-11-16T10:25:13,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/145f63603d414a548e1572ab5132b06f to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/145f63603d414a548e1572ab5132b06f 2024-11-16T10:25:13,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/a35810f51bbc4ef88da4060895b5ea80 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/a35810f51bbc4ef88da4060895b5ea80 2024-11-16T10:25:13,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/8d05420950994a03badd5bdf5be6a450 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/8d05420950994a03badd5bdf5be6a450 2024-11-16T10:25:13,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/04b6b0d40a4a450593ea3be60dc6749e to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/04b6b0d40a4a450593ea3be60dc6749e 2024-11-16T10:25:13,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/0351adc88b994c80825f2e817c327573 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/0351adc88b994c80825f2e817c327573 2024-11-16T10:25:13,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/40ee0f215787447ea2f5ef6546b6ddef to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/40ee0f215787447ea2f5ef6546b6ddef 2024-11-16T10:25:13,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/e72b6ccfc85642c6bde751c3f2fc08b1 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/e72b6ccfc85642c6bde751c3f2fc08b1 2024-11-16T10:25:13,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/67df6bcd436f464ca15b8eff760cd5e3 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/67df6bcd436f464ca15b8eff760cd5e3 2024-11-16T10:25:13,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/a9a67a2de22d41b4a7d9c4b6f640c635 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/a9a67a2de22d41b4a7d9c4b6f640c635 2024-11-16T10:25:13,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/b176345f0aa74a53a2a6f735849c7975 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/b176345f0aa74a53a2a6f735849c7975 2024-11-16T10:25:13,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/00c3e817ac1a41c0abf16ec3ecb79539 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/00c3e817ac1a41c0abf16ec3ecb79539 2024-11-16T10:25:13,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/7fbefd9344e444df9705254048051f50 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/7fbefd9344e444df9705254048051f50 2024-11-16T10:25:13,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/7557a20a9f0e4adcabf8e1a0dbf48726 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/7557a20a9f0e4adcabf8e1a0dbf48726 2024-11-16T10:25:13,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/13971819814a46ba91b3cb1be4741917 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/13971819814a46ba91b3cb1be4741917 2024-11-16T10:25:13,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/bbd63ceaee314a10b98d39d2b3fcfd63 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/bbd63ceaee314a10b98d39d2b3fcfd63 2024-11-16T10:25:13,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/7426f6762a9b4ff989577b7de19a1f49 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/7426f6762a9b4ff989577b7de19a1f49 2024-11-16T10:25:13,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/4929f5bde85c4a749f379d479273e44f to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/4929f5bde85c4a749f379d479273e44f 2024-11-16T10:25:13,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/e71cc1bd31a441a9a6792cd0082100f8 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/e71cc1bd31a441a9a6792cd0082100f8 2024-11-16T10:25:13,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/24dbf127abe6486d8e953cedbfe57e6d to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/24dbf127abe6486d8e953cedbfe57e6d 2024-11-16T10:25:13,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/99b74d1986824c73b7b51d9f7ee881a9 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/99b74d1986824c73b7b51d9f7ee881a9 2024-11-16T10:25:13,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/1768ad386c3949b89b3eb7602f6d0b9e to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/1768ad386c3949b89b3eb7602f6d0b9e 2024-11-16T10:25:13,536 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/recovered.edits/446.seqid, newMaxSeqId=446, maxSeqId=1 2024-11-16T10:25:13,536 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. 2024-11-16T10:25:13,536 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for d41ecfe9dd6b68876325d42387a8c316: Waiting for close lock at 1731752712160Running coprocessor pre-close hooks at 1731752712160Disabling compacts and flushes for region at 1731752712160Disabling writes for close at 1731752712175 (+15 ms)Obtaining lock to block concurrent updates at 1731752712175Preparing flush snapshotting stores in d41ecfe9dd6b68876325d42387a8c316 at 1731752712175Finished memstore snapshotting TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=18720, getOffHeapSize=0, getCellsCount=150 at 1731752712175Flushing stores of TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316. at 1731752712175Flushing d41ecfe9dd6b68876325d42387a8c316/A: creating writer at 1731752712175Flushing d41ecfe9dd6b68876325d42387a8c316/A: appending metadata at 1731752712178 (+3 ms)Flushing d41ecfe9dd6b68876325d42387a8c316/A: closing flushed file at 1731752712178Flushing d41ecfe9dd6b68876325d42387a8c316/B: creating writer at 1731752712594 (+416 ms)Flushing d41ecfe9dd6b68876325d42387a8c316/B: appending metadata at 1731752712597 (+3 ms)Flushing d41ecfe9dd6b68876325d42387a8c316/B: closing flushed file at 1731752712597Flushing d41ecfe9dd6b68876325d42387a8c316/C: creating writer at 1731752713012 (+415 ms)Flushing d41ecfe9dd6b68876325d42387a8c316/C: appending metadata at 1731752713016 (+4 ms)Flushing d41ecfe9dd6b68876325d42387a8c316/C: closing flushed file at 1731752713016Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c60fb99: reopening flushed file at 1731752713434 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19da3c8e: reopening flushed file at 1731752713441 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29d1b662: reopening flushed file at 1731752713446 (+5 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for d41ecfe9dd6b68876325d42387a8c316 in 1278ms, sequenceid=443, compaction requested=true at 1731752713453 (+7 ms)Writing region close event to WAL at 1731752713533 (+80 ms)Running coprocessor post-close hooks at 1731752713536 (+3 ms)Closed at 1731752713536 2024-11-16T10:25:13,538 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:25:13,538 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=d41ecfe9dd6b68876325d42387a8c316, regionState=CLOSED 2024-11-16T10:25:13,539 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 because future has completed 2024-11-16T10:25:13,541 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-16T10:25:13,541 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure d41ecfe9dd6b68876325d42387a8c316, server=b16ff38458c1,35873,1731752627506 in 1.5340 sec 2024-11-16T10:25:13,542 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-16T10:25:13,542 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d41ecfe9dd6b68876325d42387a8c316, UNASSIGN in 1.5390 sec 2024-11-16T10:25:13,544 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-16T10:25:13,544 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5430 sec 2024-11-16T10:25:13,545 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752713545"}]},"ts":"1731752713545"} 2024-11-16T10:25:13,546 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T10:25:13,546 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T10:25:13,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5540 sec 2024-11-16T10:25:14,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T10:25:14,148 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:25:14,150 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T10:25:14,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:25:14,155 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:25:14,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-16T10:25:14,156 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:25:14,160 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:25:14,164 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A, FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B, FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C, FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/recovered.edits] 2024-11-16T10:25:14,169 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/6bab9b3ad01346d49059ca8622d281de to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/6bab9b3ad01346d49059ca8622d281de 2024-11-16T10:25:14,170 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/8156c0a74ef7486db97998d52f423a3b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/A/8156c0a74ef7486db97998d52f423a3b 2024-11-16T10:25:14,174 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/3c828667a1c542ef99098c1db14eedc5 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/3c828667a1c542ef99098c1db14eedc5 2024-11-16T10:25:14,176 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/e9207c1cc9b943ce9a2ef672bb10ae75 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/B/e9207c1cc9b943ce9a2ef672bb10ae75 2024-11-16T10:25:14,178 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/40afe031c23b4ede883e5b52a465d4d6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/40afe031c23b4ede883e5b52a465d4d6 2024-11-16T10:25:14,180 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/5a78b3dd511d41ca86ba7080581e4ae7 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/5a78b3dd511d41ca86ba7080581e4ae7 2024-11-16T10:25:14,181 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/e84fb3fc45764573b4348e1fba59e9d8 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/e84fb3fc45764573b4348e1fba59e9d8 2024-11-16T10:25:14,182 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/eb547d893e7643d18633ed3b262a613a to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/C/eb547d893e7643d18633ed3b262a613a 2024-11-16T10:25:14,185 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/recovered.edits/446.seqid to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316/recovered.edits/446.seqid 2024-11-16T10:25:14,186 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/d41ecfe9dd6b68876325d42387a8c316 2024-11-16T10:25:14,186 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T10:25:14,189 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:25:14,194 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T10:25:14,197 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T10:25:14,199 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:25:14,199 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T10:25:14,199 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731752714199"}]},"ts":"9223372036854775807"} 2024-11-16T10:25:14,201 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T10:25:14,201 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d41ecfe9dd6b68876325d42387a8c316, NAME => 'TestAcidGuarantees,,1731752686215.d41ecfe9dd6b68876325d42387a8c316.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T10:25:14,202 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T10:25:14,202 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731752714202"}]},"ts":"9223372036854775807"} 2024-11-16T10:25:14,204 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T10:25:14,205 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:25:14,206 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 55 msec 2024-11-16T10:25:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-16T10:25:14,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T10:25:14,267 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:25:14,281 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=244 (was 241) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1713545760_22 at /127.0.0.1:38114 [Waiting for operation #236] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107388350_22 at /127.0.0.1:41688 [Waiting for operation #923] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107388350_22 at /127.0.0.1:46296 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1713545760_22 at /127.0.0.1:46300 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-107388350_22 at /127.0.0.1:46284 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1713545760_22 at /127.0.0.1:46274 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=469 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=481 (was 497), ProcessCount=11 (was 11), AvailableMemoryMB=4199 (was 4077) - AvailableMemoryMB LEAK? - 2024-11-16T10:25:14,292 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=244, OpenFileDescriptor=469, MaxFileDescriptor=1048576, SystemLoadAverage=481, ProcessCount=11, AvailableMemoryMB=4199 2024-11-16T10:25:14,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T10:25:14,294 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T10:25:14,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T10:25:14,296 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T10:25:14,296 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:14,297 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-11-16T10:25:14,297 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T10:25:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-16T10:25:14,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742119_1295 (size=959) 2024-11-16T10:25:14,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-16T10:25:14,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-16T10:25:14,710 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7 2024-11-16T10:25:14,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742120_1296 (size=53) 2024-11-16T10:25:14,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-16T10:25:15,096 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T10:25:15,120 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:25:15,120 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a591075eabfc58d555a22f619c4575e6, disabling compactions & flushes 2024-11-16T10:25:15,120 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:15,120 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:15,120 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. after waiting 0 ms 2024-11-16T10:25:15,120 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:15,120 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:15,120 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a591075eabfc58d555a22f619c4575e6: Waiting for close lock at 1731752715120Disabling compacts and flushes for region at 1731752715120Disabling writes for close at 1731752715120Writing region close event to WAL at 1731752715120Closed at 1731752715120 2024-11-16T10:25:15,122 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T10:25:15,122 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731752715122"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731752715122"}]},"ts":"1731752715122"} 2024-11-16T10:25:15,123 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T10:25:15,124 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T10:25:15,124 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752715124"}]},"ts":"1731752715124"} 2024-11-16T10:25:15,126 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T10:25:15,126 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a591075eabfc58d555a22f619c4575e6, ASSIGN}] 2024-11-16T10:25:15,127 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a591075eabfc58d555a22f619c4575e6, ASSIGN 2024-11-16T10:25:15,128 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a591075eabfc58d555a22f619c4575e6, ASSIGN; state=OFFLINE, location=b16ff38458c1,35873,1731752627506; forceNewPlan=false, retain=false 2024-11-16T10:25:15,279 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=a591075eabfc58d555a22f619c4575e6, regionState=OPENING, regionLocation=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:15,280 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a591075eabfc58d555a22f619c4575e6, ASSIGN because future has completed 2024-11-16T10:25:15,281 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506}] 2024-11-16T10:25:15,435 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:15,435 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} 2024-11-16T10:25:15,436 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:15,436 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:25:15,436 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:15,436 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-16T10:25:15,437 INFO [StoreOpener-a591075eabfc58d555a22f619c4575e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:15,439 INFO [StoreOpener-a591075eabfc58d555a22f619c4575e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a591075eabfc58d555a22f619c4575e6 columnFamilyName A 2024-11-16T10:25:15,439 DEBUG [StoreOpener-a591075eabfc58d555a22f619c4575e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:15,440 INFO [StoreOpener-a591075eabfc58d555a22f619c4575e6-1 {}] regionserver.HStore(327): Store=a591075eabfc58d555a22f619c4575e6/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:25:15,440 INFO [StoreOpener-a591075eabfc58d555a22f619c4575e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:15,441 INFO [StoreOpener-a591075eabfc58d555a22f619c4575e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a591075eabfc58d555a22f619c4575e6 columnFamilyName B 2024-11-16T10:25:15,441 DEBUG [StoreOpener-a591075eabfc58d555a22f619c4575e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:15,442 INFO [StoreOpener-a591075eabfc58d555a22f619c4575e6-1 {}] regionserver.HStore(327): Store=a591075eabfc58d555a22f619c4575e6/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:25:15,442 INFO [StoreOpener-a591075eabfc58d555a22f619c4575e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:15,443 INFO [StoreOpener-a591075eabfc58d555a22f619c4575e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a591075eabfc58d555a22f619c4575e6 columnFamilyName C 2024-11-16T10:25:15,443 DEBUG [StoreOpener-a591075eabfc58d555a22f619c4575e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:15,444 INFO [StoreOpener-a591075eabfc58d555a22f619c4575e6-1 {}] regionserver.HStore(327): Store=a591075eabfc58d555a22f619c4575e6/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:25:15,444 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:15,444 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:15,445 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:15,446 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:15,446 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:15,447 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T10:25:15,448 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:15,450 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T10:25:15,450 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened a591075eabfc58d555a22f619c4575e6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65694985, jitterRate=-0.02106843888759613}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T10:25:15,450 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:15,451 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for a591075eabfc58d555a22f619c4575e6: Running coprocessor pre-open hook at 1731752715436Writing region info on filesystem at 1731752715436Initializing all the Stores at 1731752715437 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752715437Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752715437Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752715437Cleaning up temporary data from old regions at 1731752715446 (+9 ms)Running coprocessor post-open hooks at 1731752715450 (+4 ms)Region opened successfully at 1731752715451 (+1 ms) 2024-11-16T10:25:15,451 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., pid=89, masterSystemTime=1731752715433 2024-11-16T10:25:15,453 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:15,453 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:15,454 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=a591075eabfc58d555a22f619c4575e6, regionState=OPEN, openSeqNum=2, regionLocation=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:15,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 because future has completed 2024-11-16T10:25:15,457 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-16T10:25:15,457 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 in 175 msec 2024-11-16T10:25:15,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-16T10:25:15,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a591075eabfc58d555a22f619c4575e6, ASSIGN in 331 msec 2024-11-16T10:25:15,460 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T10:25:15,460 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752715460"}]},"ts":"1731752715460"} 2024-11-16T10:25:15,461 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T10:25:15,461 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T10:25:15,462 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1670 sec 2024-11-16T10:25:16,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-16T10:25:16,456 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:25:16,457 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d3de22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:16,457 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:25:16,457 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:25:16,458 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:25:16,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:25:16,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:25:16,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@192b375a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:16,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:25:16,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:25:16,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:16,460 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:25:16,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58d80c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:16,460 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:25:16,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:25:16,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:25:16,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:25:16,462 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T10:25:16,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is b16ff38458c1,38809,1731752626047 2024-11-16T10:25:16,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6d3ee8ba 2024-11-16T10:25:16,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T10:25:16,464 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T10:25:16,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T10:25:16,465 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T10:25:16,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-16T10:25:16,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742121_1297 (size=995) 2024-11-16T10:25:16,878 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-16T10:25:16,878 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-16T10:25:16,880 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T10:25:16,882 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a591075eabfc58d555a22f619c4575e6, REOPEN/MOVE}] 2024-11-16T10:25:16,883 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a591075eabfc58d555a22f619c4575e6, REOPEN/MOVE 2024-11-16T10:25:16,884 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=a591075eabfc58d555a22f619c4575e6, regionState=CLOSING, regionLocation=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:16,885 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a591075eabfc58d555a22f619c4575e6, REOPEN/MOVE because future has completed 2024-11-16T10:25:16,885 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T10:25:16,885 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506}] 2024-11-16T10:25:17,037 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:17,038 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T10:25:17,038 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing a591075eabfc58d555a22f619c4575e6, disabling compactions & flushes 2024-11-16T10:25:17,038 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:17,038 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:17,038 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. after waiting 0 ms 2024-11-16T10:25:17,038 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:17,042 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T10:25:17,043 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:17,043 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for a591075eabfc58d555a22f619c4575e6: Waiting for close lock at 1731752717038Running coprocessor pre-close hooks at 1731752717038Disabling compacts and flushes for region at 1731752717038Disabling writes for close at 1731752717038Writing region close event to WAL at 1731752717038Running coprocessor post-close hooks at 1731752717043 (+5 ms)Closed at 1731752717043 2024-11-16T10:25:17,043 WARN [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: a591075eabfc58d555a22f619c4575e6 to self. 2024-11-16T10:25:17,046 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:17,046 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=a591075eabfc58d555a22f619c4575e6, regionState=CLOSED 2024-11-16T10:25:17,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 because future has completed 2024-11-16T10:25:17,053 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-16T10:25:17,053 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 in 165 msec 2024-11-16T10:25:17,054 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a591075eabfc58d555a22f619c4575e6, REOPEN/MOVE; state=CLOSED, location=b16ff38458c1,35873,1731752627506; forceNewPlan=false, retain=true 2024-11-16T10:25:17,204 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=a591075eabfc58d555a22f619c4575e6, regionState=OPENING, regionLocation=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:17,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a591075eabfc58d555a22f619c4575e6, REOPEN/MOVE because future has completed 2024-11-16T10:25:17,206 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506}] 2024-11-16T10:25:17,359 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:17,359 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} 2024-11-16T10:25:17,360 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:17,360 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:25:17,360 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:17,360 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:17,361 INFO [StoreOpener-a591075eabfc58d555a22f619c4575e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:17,362 INFO [StoreOpener-a591075eabfc58d555a22f619c4575e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a591075eabfc58d555a22f619c4575e6 columnFamilyName A 2024-11-16T10:25:17,363 DEBUG [StoreOpener-a591075eabfc58d555a22f619c4575e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:17,363 INFO [StoreOpener-a591075eabfc58d555a22f619c4575e6-1 {}] regionserver.HStore(327): Store=a591075eabfc58d555a22f619c4575e6/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:25:17,363 INFO [StoreOpener-a591075eabfc58d555a22f619c4575e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:17,364 INFO [StoreOpener-a591075eabfc58d555a22f619c4575e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a591075eabfc58d555a22f619c4575e6 columnFamilyName B 2024-11-16T10:25:17,364 DEBUG [StoreOpener-a591075eabfc58d555a22f619c4575e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:17,365 INFO [StoreOpener-a591075eabfc58d555a22f619c4575e6-1 {}] regionserver.HStore(327): Store=a591075eabfc58d555a22f619c4575e6/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:25:17,365 INFO [StoreOpener-a591075eabfc58d555a22f619c4575e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:17,365 INFO [StoreOpener-a591075eabfc58d555a22f619c4575e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a591075eabfc58d555a22f619c4575e6 columnFamilyName C 2024-11-16T10:25:17,365 DEBUG [StoreOpener-a591075eabfc58d555a22f619c4575e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:17,366 INFO [StoreOpener-a591075eabfc58d555a22f619c4575e6-1 {}] regionserver.HStore(327): Store=a591075eabfc58d555a22f619c4575e6/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:25:17,366 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:17,366 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:17,367 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:17,368 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:17,368 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:17,368 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T10:25:17,369 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:17,370 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened a591075eabfc58d555a22f619c4575e6; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66359608, jitterRate=-0.01116478443145752}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T10:25:17,370 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:17,371 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for a591075eabfc58d555a22f619c4575e6: Running coprocessor pre-open hook at 1731752717360Writing region info on filesystem at 1731752717360Initializing all the Stores at 1731752717361 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752717361Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752717361Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752717361Cleaning up temporary data from old regions at 1731752717368 (+7 ms)Running coprocessor post-open hooks at 1731752717370 (+2 ms)Region opened successfully at 1731752717371 (+1 ms) 2024-11-16T10:25:17,372 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., pid=94, masterSystemTime=1731752717357 2024-11-16T10:25:17,374 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:17,374 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:17,374 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=a591075eabfc58d555a22f619c4575e6, regionState=OPEN, openSeqNum=5, regionLocation=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:17,376 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 because future has completed 2024-11-16T10:25:17,378 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-11-16T10:25:17,378 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 in 170 msec 2024-11-16T10:25:17,380 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-16T10:25:17,380 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a591075eabfc58d555a22f619c4575e6, REOPEN/MOVE in 496 msec 2024-11-16T10:25:17,382 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-16T10:25:17,382 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 500 msec 2024-11-16T10:25:17,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 916 msec 2024-11-16T10:25:17,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-16T10:25:17,384 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:25:17,385 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,385 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:25:17,385 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:25:17,386 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:25:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:25:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:25:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49650aa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:25:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:25:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:17,387 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:25:17,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5733048b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,388 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,388 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:25:17,388 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:25:17,389 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:25:17,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:25:17,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:25:17,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78d8558e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:25:17,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:25:17,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:17,390 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:25:17,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ca52c26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,391 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,391 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:25:17,391 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:25:17,392 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:25:17,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:25:17,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:25:17,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70aba824, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:25:17,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:25:17,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:17,393 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:25:17,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76c12111, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,394 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e71a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,394 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:25:17,394 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:25:17,395 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:25:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:25:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:25:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b811c27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:25:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:25:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:17,396 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:25:17,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61191ca6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,397 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b9dbe2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,397 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:25:17,397 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:25:17,398 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:25:17,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:25:17,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:25:17,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18c5d415, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:25:17,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:25:17,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:17,399 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:25:17,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36a0d86c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,400 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,400 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:25:17,400 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:25:17,401 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:25:17,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:25:17,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:25:17,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5edcb367, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:25:17,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:25:17,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:17,402 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:25:17,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28580e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,403 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,403 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:25:17,403 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:25:17,404 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:25:17,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:25:17,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:25:17,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@516c232f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:25:17,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:25:17,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:17,405 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:25:17,406 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b458b6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,406 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:25:17,406 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:25:17,407 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:25:17,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:25:17,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:25:17,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69fc40f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:25:17,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:25:17,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:17,408 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:25:17,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ee52a75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,410 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180bd544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,410 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:25:17,410 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:25:17,411 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:25:17,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:25:17,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:25:17,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@307aadbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:25:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:25:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:17,413 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:25:17,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f3cdfbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d553196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,414 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:25:17,414 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:25:17,415 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:25:17,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:25:17,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:25:17,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15dad1c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:25:17,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:25:17,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:17,416 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:25:17,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63cb8e59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:17,417 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:25:17,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-16T10:25:17,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T10:25:17,419 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:25:17,419 DEBUG [Thread-1392 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:25:17,419 DEBUG [Thread-1377 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:25:17,420 DEBUG [Thread-1394 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:25:17,420 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:25:17,420 DEBUG [Thread-1390 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:25:17,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:25:17,420 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:25:17,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:25:17,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:25:17,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:25:17,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:25:17,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:25:17,421 DEBUG [Thread-1388 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:25:17,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:25:17,422 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:25:17,422 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:25:17,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:25:17,422 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:25:17,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:25:17,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:25:17,423 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:25:17,425 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:25:17,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5] 2024-11-16T10:25:17,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5] 2024-11-16T10:25:17,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5] 2024-11-16T10:25:17,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5] 2024-11-16T10:25:17,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5] 2024-11-16T10:25:17,432 DEBUG [Thread-1396 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:25:17,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:25:17,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:25:17,433 DEBUG [Thread-1385 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:25:17,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:25:17,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:25:17,436 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:25:17,436 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:25:17,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5] 2024-11-16T10:25:17,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5] 2024-11-16T10:25:17,438 DEBUG [Thread-1379 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:25:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:25:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:25:17,443 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58534, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:25:17,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5] 2024-11-16T10:25:17,445 DEBUG [Thread-1381 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:25:17,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:25:17,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:25:17,446 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:25:17,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5] 2024-11-16T10:25:17,450 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:25:17,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a591075eabfc58d555a22f619c4575e6 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T10:25:17,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:17,460 DEBUG [Thread-1383 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:25:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:25:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:25:17,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:25:17,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5] 2024-11-16T10:25:17,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111620e2b783539f437f9b70a272a074500f_a591075eabfc58d555a22f619c4575e6 is 50, key is test_row_0/A:col10/1731752717442/Put/seqid=0 2024-11-16T10:25:17,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742122_1298 (size=12154) 2024-11-16T10:25:17,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:17,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752777507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:17,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:17,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752777508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:17,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:17,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:17,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:17,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752777510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:17,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:17,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752777510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:17,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:17,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:17,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:17,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752777513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:17,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:17,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T10:25:17,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T10:25:17,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:17,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:17,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:17,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:17,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:17,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:17,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:17,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752777626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:17,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:17,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:17,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752777626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:17,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:17,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752777626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:17,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752777626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:17,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:17,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752777626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:17,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:17,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T10:25:17,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:17,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:17,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:17,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:17,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:17,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:17,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T10:25:17,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:17,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752777836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:17,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:17,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752777836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:17,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:17,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752777837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:17,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:17,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752777837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752777837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:17,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T10:25:17,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:17,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:17,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:17,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:17,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:17,905 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:17,910 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111620e2b783539f437f9b70a272a074500f_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111620e2b783539f437f9b70a272a074500f_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:17,913 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/92f608ec54c74032b24878b046f302b9, store: [table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:17,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/92f608ec54c74032b24878b046f302b9 is 175, key is test_row_0/A:col10/1731752717442/Put/seqid=0 2024-11-16T10:25:17,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742123_1299 (size=30955) 2024-11-16T10:25:17,962 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/92f608ec54c74032b24878b046f302b9 2024-11-16T10:25:17,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/16c4dcac7b6141ceb56b27d9d38805a0 is 50, key is test_row_0/B:col10/1731752717442/Put/seqid=0 2024-11-16T10:25:18,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742124_1300 (size=12001) 2024-11-16T10:25:18,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T10:25:18,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:18,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:18,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:18,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:18,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:18,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:18,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T10:25:18,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:18,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752778146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:18,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:18,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752778146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:18,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:18,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752778146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:18,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:18,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752778156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:18,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:18,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752778156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:18,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:18,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:18,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T10:25:18,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:18,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:18,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:18,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:18,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:18,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T10:25:18,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:18,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:18,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:18,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:18,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:18,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/16c4dcac7b6141ceb56b27d9d38805a0 2024-11-16T10:25:18,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/62d408e98454446ebb02ef5388296b92 is 50, key is test_row_0/C:col10/1731752717442/Put/seqid=0 2024-11-16T10:25:18,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742125_1301 (size=12001) 2024-11-16T10:25:18,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/62d408e98454446ebb02ef5388296b92 2024-11-16T10:25:18,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/92f608ec54c74032b24878b046f302b9 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/92f608ec54c74032b24878b046f302b9 2024-11-16T10:25:18,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/92f608ec54c74032b24878b046f302b9, entries=150, sequenceid=16, filesize=30.2 K 2024-11-16T10:25:18,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/16c4dcac7b6141ceb56b27d9d38805a0 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/16c4dcac7b6141ceb56b27d9d38805a0 2024-11-16T10:25:18,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/16c4dcac7b6141ceb56b27d9d38805a0, entries=150, sequenceid=16, filesize=11.7 K 2024-11-16T10:25:18,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/62d408e98454446ebb02ef5388296b92 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/62d408e98454446ebb02ef5388296b92 2024-11-16T10:25:18,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T10:25:18,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:18,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:18,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:18,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:18,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:18,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:18,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/62d408e98454446ebb02ef5388296b92, entries=150, sequenceid=16, filesize=11.7 K 2024-11-16T10:25:18,517 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for a591075eabfc58d555a22f619c4575e6 in 1067ms, sequenceid=16, compaction requested=false 2024-11-16T10:25:18,517 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-16T10:25:18,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T10:25:18,667 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:25:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:18,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a591075eabfc58d555a22f619c4575e6 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T10:25:18,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T10:25:18,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:18,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:18,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:18,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:18,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:18,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116eb8b0d7faa1941d1b75ba66064fc708d_a591075eabfc58d555a22f619c4575e6 is 50, key is test_row_0/A:col10/1731752717506/Put/seqid=0 2024-11-16T10:25:18,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752778695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:18,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752778694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:18,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:18,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752778701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:18,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:18,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752778701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:18,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:18,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:18,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752778701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:18,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:18,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:18,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742126_1302 (size=14594) 2024-11-16T10:25:18,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:18,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752778806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:18,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752778806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:18,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T10:25:18,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:18,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:18,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:18,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:18,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:18,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752778817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:18,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:18,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:18,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752778817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:18,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752778817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:18,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:18,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:18,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:18,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:18,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T10:25:18,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:18,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:18,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:18,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:18,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:19,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752779016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:19,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:19,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752779016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:19,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:19,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752779036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:19,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752779037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:19,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:19,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:19,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752779037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:19,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:19,074 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T10:25:19,135 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:19,139 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116eb8b0d7faa1941d1b75ba66064fc708d_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116eb8b0d7faa1941d1b75ba66064fc708d_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:19,140 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/46fbe02d9beb4e9989448dcd40213f23, store: [table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:19,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/46fbe02d9beb4e9989448dcd40213f23 is 175, key is test_row_0/A:col10/1731752717506/Put/seqid=0 2024-11-16T10:25:19,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T10:25:19,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:19,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:19,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:19,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:19,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:19,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742127_1303 (size=39549) 2024-11-16T10:25:19,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T10:25:19,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:19,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:19,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:19,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:19,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:19,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:19,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:19,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752779336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:19,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:19,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752779336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:19,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:19,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:19,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:19,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752779346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:19,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:19,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752779346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:19,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:19,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:19,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752779357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:19,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T10:25:19,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:19,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:19,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:19,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:19,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:19,552 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/46fbe02d9beb4e9989448dcd40213f23 2024-11-16T10:25:19,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/0be6c4c45b234533a55c9631987fac4e is 50, key is test_row_0/B:col10/1731752717506/Put/seqid=0 2024-11-16T10:25:19,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T10:25:19,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742128_1304 (size=12001) 2024-11-16T10:25:19,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T10:25:19,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:19,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:19,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:19,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:19,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:19,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:19,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T10:25:19,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:19,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:19,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:19,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:19,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:19,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:19,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:19,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752779846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:19,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:19,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752779846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:19,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:19,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:19,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:19,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752779867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:19,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:19,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752779867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:19,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:19,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:19,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752779876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:19,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:19,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T10:25:19,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:19,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:19,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:19,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:19,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:19,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:19,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/0be6c4c45b234533a55c9631987fac4e 2024-11-16T10:25:20,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/2207e4f531a84edda995e630f4f44fe4 is 50, key is test_row_0/C:col10/1731752717506/Put/seqid=0 2024-11-16T10:25:20,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742129_1305 (size=12001) 2024-11-16T10:25:20,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/2207e4f531a84edda995e630f4f44fe4 2024-11-16T10:25:20,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/46fbe02d9beb4e9989448dcd40213f23 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/46fbe02d9beb4e9989448dcd40213f23 2024-11-16T10:25:20,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/46fbe02d9beb4e9989448dcd40213f23, entries=200, sequenceid=41, filesize=38.6 K 2024-11-16T10:25:20,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/0be6c4c45b234533a55c9631987fac4e as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/0be6c4c45b234533a55c9631987fac4e 2024-11-16T10:25:20,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/0be6c4c45b234533a55c9631987fac4e, entries=150, sequenceid=41, filesize=11.7 K 2024-11-16T10:25:20,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/2207e4f531a84edda995e630f4f44fe4 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/2207e4f531a84edda995e630f4f44fe4 2024-11-16T10:25:20,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/2207e4f531a84edda995e630f4f44fe4, entries=150, sequenceid=41, filesize=11.7 K 2024-11-16T10:25:20,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a591075eabfc58d555a22f619c4575e6 in 1380ms, sequenceid=41, compaction requested=false 2024-11-16T10:25:20,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:20,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T10:25:20,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:20,073 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing a591075eabfc58d555a22f619c4575e6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T10:25:20,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167212da567e00473cad10208549a2259f_a591075eabfc58d555a22f619c4575e6 is 50, key is test_row_0/A:col10/1731752718699/Put/seqid=0 2024-11-16T10:25:20,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742130_1306 (size=12154) 2024-11-16T10:25:20,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:20,532 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167212da567e00473cad10208549a2259f_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167212da567e00473cad10208549a2259f_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:20,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/32baae1b021d43c68a13aa5cabb67e83, store: [table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:20,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/32baae1b021d43c68a13aa5cabb67e83 is 175, key is test_row_0/A:col10/1731752718699/Put/seqid=0 2024-11-16T10:25:20,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742131_1307 (size=30955) 2024-11-16T10:25:20,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:20,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:20,945 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/32baae1b021d43c68a13aa5cabb67e83 2024-11-16T10:25:20,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752780933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:20,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:20,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:20,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:20,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:20,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752780935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752780936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:20,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752780942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:20,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/80dd666d473b41e5a82545942b9ddd4f is 50, key is test_row_0/B:col10/1731752718699/Put/seqid=0 2024-11-16T10:25:20,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752780956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:20,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:20,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:20,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:21,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742132_1308 (size=12001) 2024-11-16T10:25:21,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:21,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752781066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:21,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:21,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752781066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:21,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:21,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:21,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752781066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:21,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:21,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:21,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752781066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:21,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:21,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:21,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:21,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752781076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:21,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:21,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:21,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752781291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:21,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:21,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:21,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752781291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:21,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:21,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:21,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752781291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:21,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:21,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:21,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752781295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:21,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:21,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:21,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752781296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:21,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:21,417 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/80dd666d473b41e5a82545942b9ddd4f 2024-11-16T10:25:21,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/d3b90a0ca5b84011abee5ade25fb6ae3 is 50, key is test_row_0/C:col10/1731752718699/Put/seqid=0 2024-11-16T10:25:21,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742133_1309 (size=12001) 2024-11-16T10:25:21,512 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/d3b90a0ca5b84011abee5ade25fb6ae3 2024-11-16T10:25:21,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/32baae1b021d43c68a13aa5cabb67e83 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/32baae1b021d43c68a13aa5cabb67e83 2024-11-16T10:25:21,535 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/32baae1b021d43c68a13aa5cabb67e83, entries=150, sequenceid=52, filesize=30.2 K 2024-11-16T10:25:21,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/80dd666d473b41e5a82545942b9ddd4f as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/80dd666d473b41e5a82545942b9ddd4f 2024-11-16T10:25:21,540 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/80dd666d473b41e5a82545942b9ddd4f, entries=150, sequenceid=52, filesize=11.7 K 2024-11-16T10:25:21,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/d3b90a0ca5b84011abee5ade25fb6ae3 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/d3b90a0ca5b84011abee5ade25fb6ae3 2024-11-16T10:25:21,549 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/d3b90a0ca5b84011abee5ade25fb6ae3, entries=150, sequenceid=52, filesize=11.7 K 2024-11-16T10:25:21,550 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a591075eabfc58d555a22f619c4575e6 in 1477ms, sequenceid=52, compaction requested=true 2024-11-16T10:25:21,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:21,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:21,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-16T10:25:21,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-16T10:25:21,556 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-16T10:25:21,556 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.1320 sec 2024-11-16T10:25:21,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 4.1380 sec 2024-11-16T10:25:21,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T10:25:21,577 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:25:21,577 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:25:21,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-16T10:25:21,579 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:25:21,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T10:25:21,580 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:25:21,580 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:25:21,621 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:25:21,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a591075eabfc58d555a22f619c4575e6 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-16T10:25:21,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:21,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167aa8de3de2304be489571dc663ae3fc0_a591075eabfc58d555a22f619c4575e6 is 50, key is test_row_0/A:col10/1731752720934/Put/seqid=0 2024-11-16T10:25:21,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:21,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752781629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:21,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:21,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752781631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:21,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:21,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:21,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:21,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752781653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:21,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:21,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:21,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752781655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:21,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:21,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752781659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:21,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:21,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:21,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T10:25:21,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742134_1310 (size=12154) 2024-11-16T10:25:21,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T10:25:21,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:21,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:21,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:21,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:21,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:21,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:21,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752781766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:21,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:21,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752781766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:21,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:21,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:21,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752781778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:21,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752781786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:21,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752781786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:21,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:21,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:21,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T10:25:21,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:21,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:21,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:21,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:21,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T10:25:21,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752781986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:21,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752781986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:21,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:21,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:21,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:22,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752781997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:22,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752781997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:22,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752781997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:22,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:22,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:22,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:22,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:22,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:22,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:22,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T10:25:22,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:22,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:22,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:22,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:22,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:22,090 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:22,100 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167aa8de3de2304be489571dc663ae3fc0_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167aa8de3de2304be489571dc663ae3fc0_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:22,101 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/36c1475b40334d879c2041ece5c4225f, store: [table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:22,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/36c1475b40334d879c2041ece5c4225f is 175, key is test_row_0/A:col10/1731752720934/Put/seqid=0 2024-11-16T10:25:22,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742135_1311 (size=30955) 2024-11-16T10:25:22,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T10:25:22,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:22,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:22,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:22,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:22,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:22,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T10:25:22,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752782306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:22,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:22,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752782306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:22,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:22,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:22,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:22,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:22,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:22,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752782316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:22,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752782317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:22,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752782317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:22,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T10:25:22,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:22,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:22,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:22,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:22,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:22,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T10:25:22,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:22,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:22,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:22,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:22,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:22,539 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/36c1475b40334d879c2041ece5c4225f 2024-11-16T10:25:22,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/57c5e1e228354f79a1fa4e8d4a0d7efb is 50, key is test_row_0/B:col10/1731752720934/Put/seqid=0 2024-11-16T10:25:22,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742136_1312 (size=12001) 2024-11-16T10:25:22,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T10:25:22,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:22,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:22,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:22,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:22,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T10:25:22,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T10:25:22,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:22,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:22,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:22,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:22,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:22,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752782827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:22,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752782827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:22,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:22,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:22,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:22,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:22,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:22,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:22,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752782837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:22,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752782838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:22,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752782838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:22,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T10:25:22,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:22,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:22,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:22,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:22,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:23,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/57c5e1e228354f79a1fa4e8d4a0d7efb 2024-11-16T10:25:23,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/48b97f641bea4e9cbe133e8c31604dd8 is 50, key is test_row_0/C:col10/1731752720934/Put/seqid=0 2024-11-16T10:25:23,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742137_1313 (size=12001) 2024-11-16T10:25:23,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T10:25:23,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:23,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:23,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:23,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:23,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:23,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T10:25:23,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:23,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:23,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:23,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:23,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:23,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T10:25:23,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:23,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:23,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:23,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:23,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:23,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:23,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/48b97f641bea4e9cbe133e8c31604dd8 2024-11-16T10:25:23,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/36c1475b40334d879c2041ece5c4225f as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/36c1475b40334d879c2041ece5c4225f 2024-11-16T10:25:23,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/36c1475b40334d879c2041ece5c4225f, entries=150, sequenceid=79, filesize=30.2 K 2024-11-16T10:25:23,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/57c5e1e228354f79a1fa4e8d4a0d7efb as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/57c5e1e228354f79a1fa4e8d4a0d7efb 2024-11-16T10:25:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/57c5e1e228354f79a1fa4e8d4a0d7efb, entries=150, sequenceid=79, filesize=11.7 K 2024-11-16T10:25:23,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/48b97f641bea4e9cbe133e8c31604dd8 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/48b97f641bea4e9cbe133e8c31604dd8 2024-11-16T10:25:23,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/48b97f641bea4e9cbe133e8c31604dd8, entries=150, sequenceid=79, filesize=11.7 K 2024-11-16T10:25:23,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for a591075eabfc58d555a22f619c4575e6 in 1904ms, sequenceid=79, compaction requested=true 2024-11-16T10:25:23,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:23,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a591075eabfc58d555a22f619c4575e6:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:25:23,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:23,525 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:25:23,525 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:25:23,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a591075eabfc58d555a22f619c4575e6:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:25:23,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:23,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a591075eabfc58d555a22f619c4575e6:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:25:23,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:23,527 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132414 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:25:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,527 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): a591075eabfc58d555a22f619c4575e6/A is initiating minor compaction (all files) 2024-11-16T10:25:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,527 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a591075eabfc58d555a22f619c4575e6/A in TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:23,527 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/92f608ec54c74032b24878b046f302b9, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/46fbe02d9beb4e9989448dcd40213f23, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/32baae1b021d43c68a13aa5cabb67e83, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/36c1475b40334d879c2041ece5c4225f] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp, totalSize=129.3 K 2024-11-16T10:25:23,527 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:25:23,527 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:23,527 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. files: [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/92f608ec54c74032b24878b046f302b9, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/46fbe02d9beb4e9989448dcd40213f23, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/32baae1b021d43c68a13aa5cabb67e83, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/36c1475b40334d879c2041ece5c4225f] 2024-11-16T10:25:23,527 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): a591075eabfc58d555a22f619c4575e6/B is initiating minor compaction (all files) 2024-11-16T10:25:23,528 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a591075eabfc58d555a22f619c4575e6/B in TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:23,528 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/16c4dcac7b6141ceb56b27d9d38805a0, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/0be6c4c45b234533a55c9631987fac4e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/80dd666d473b41e5a82545942b9ddd4f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/57c5e1e228354f79a1fa4e8d4a0d7efb] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp, totalSize=46.9 K 2024-11-16T10:25:23,528 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92f608ec54c74032b24878b046f302b9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731752717442 2024-11-16T10:25:23,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,528 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 16c4dcac7b6141ceb56b27d9d38805a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731752717442 2024-11-16T10:25:23,528 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46fbe02d9beb4e9989448dcd40213f23, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731752717506 2024-11-16T10:25:23,529 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 0be6c4c45b234533a55c9631987fac4e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731752717506 2024-11-16T10:25:23,529 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32baae1b021d43c68a13aa5cabb67e83, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731752718681 2024-11-16T10:25:23,529 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 80dd666d473b41e5a82545942b9ddd4f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731752718681 2024-11-16T10:25:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,529 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36c1475b40334d879c2041ece5c4225f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731752720934 2024-11-16T10:25:23,530 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 57c5e1e228354f79a1fa4e8d4a0d7efb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731752720934 2024-11-16T10:25:23,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,549 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:23,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,571 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a591075eabfc58d555a22f619c4575e6#B#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:23,571 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/0f36fed28754404390572fa491d35756 is 50, key is test_row_0/B:col10/1731752720934/Put/seqid=0 2024-11-16T10:25:23,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,577 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116973bf53573b0460f833e804b2e27f82b_a591075eabfc58d555a22f619c4575e6 store=[table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:23,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,579 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116973bf53573b0460f833e804b2e27f82b_a591075eabfc58d555a22f619c4575e6, store=[table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:23,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,580 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116973bf53573b0460f833e804b2e27f82b_a591075eabfc58d555a22f619c4575e6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:23,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T10:25:23,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:23,586 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing a591075eabfc58d555a22f619c4575e6 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-16T10:25:23,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116abde551077b840cb9dc9a39da8504d8c_a591075eabfc58d555a22f619c4575e6 is 50, key is test_row_1/A:col10/1731752721653/Put/seqid=0 2024-11-16T10:25:23,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742138_1314 (size=12139) 2024-11-16T10:25:23,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742139_1315 (size=4469) 2024-11-16T10:25:23,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,614 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a591075eabfc58d555a22f619c4575e6#A#compaction#269 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:23,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,614 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/0b5c3d8dd19e43cab2eeeacd4bfc463e is 175, key is test_row_0/A:col10/1731752720934/Put/seqid=0 2024-11-16T10:25:23,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742140_1316 (size=9714) 2024-11-16T10:25:23,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742141_1317 (size=31093) 2024-11-16T10:25:23,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,643 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/0b5c3d8dd19e43cab2eeeacd4bfc463e as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/0b5c3d8dd19e43cab2eeeacd4bfc463e 2024-11-16T10:25:23,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,651 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a591075eabfc58d555a22f619c4575e6/A of a591075eabfc58d555a22f619c4575e6 into 0b5c3d8dd19e43cab2eeeacd4bfc463e(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:23,651 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:23,651 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., storeName=a591075eabfc58d555a22f619c4575e6/A, priority=12, startTime=1731752723525; duration=0sec 2024-11-16T10:25:23,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,651 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:23,651 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a591075eabfc58d555a22f619c4575e6:A 2024-11-16T10:25:23,651 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:25:23,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,652 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:25:23,652 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): a591075eabfc58d555a22f619c4575e6/C is initiating minor compaction (all files) 2024-11-16T10:25:23,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,660 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a591075eabfc58d555a22f619c4575e6/C in TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,660 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/62d408e98454446ebb02ef5388296b92, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/2207e4f531a84edda995e630f4f44fe4, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/d3b90a0ca5b84011abee5ade25fb6ae3, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/48b97f641bea4e9cbe133e8c31604dd8] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp, totalSize=46.9 K 2024-11-16T10:25:23,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,661 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62d408e98454446ebb02ef5388296b92, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731752717442 2024-11-16T10:25:23,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,661 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2207e4f531a84edda995e630f4f44fe4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731752717506 2024-11-16T10:25:23,662 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3b90a0ca5b84011abee5ade25fb6ae3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731752718681 2024-11-16T10:25:23,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,662 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48b97f641bea4e9cbe133e8c31604dd8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731752720934 2024-11-16T10:25:23,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,681 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a591075eabfc58d555a22f619c4575e6#C#compaction#272 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:23,681 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/80542add2d0f45deb845a1c6194350f4 is 50, key is test_row_0/C:col10/1731752720934/Put/seqid=0 2024-11-16T10:25:23,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742142_1318 (size=12139) 2024-11-16T10:25:23,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,699 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/80542add2d0f45deb845a1c6194350f4 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/80542add2d0f45deb845a1c6194350f4 2024-11-16T10:25:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,704 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a591075eabfc58d555a22f619c4575e6/C of a591075eabfc58d555a22f619c4575e6 into 80542add2d0f45deb845a1c6194350f4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:23,704 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:23,704 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., storeName=a591075eabfc58d555a22f619c4575e6/C, priority=12, startTime=1731752723526; duration=0sec 2024-11-16T10:25:23,704 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:23,704 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a591075eabfc58d555a22f619c4575e6:C 2024-11-16T10:25:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T10:25:23,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:23,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752783953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:23,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752783958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:23,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:23,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:23,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:23,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752783960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:23,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:23,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:23,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:23,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:23,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:23,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:23,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752783970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:23,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752783973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:24,019 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/0f36fed28754404390572fa491d35756 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/0f36fed28754404390572fa491d35756 2024-11-16T10:25:24,028 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a591075eabfc58d555a22f619c4575e6/B of a591075eabfc58d555a22f619c4575e6 into 0f36fed28754404390572fa491d35756(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:24,029 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:24,029 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., storeName=a591075eabfc58d555a22f619c4575e6/B, priority=12, startTime=1731752723525; duration=0sec 2024-11-16T10:25:24,029 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:24,029 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a591075eabfc58d555a22f619c4575e6:B 2024-11-16T10:25:24,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:24,043 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116abde551077b840cb9dc9a39da8504d8c_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116abde551077b840cb9dc9a39da8504d8c_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:24,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/4a7764e881914c678daa292d99d8b3da, store: [table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:24,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/4a7764e881914c678daa292d99d8b3da is 175, key is test_row_1/A:col10/1731752721653/Put/seqid=0 2024-11-16T10:25:24,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742143_1319 (size=22361) 2024-11-16T10:25:24,050 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=88, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/4a7764e881914c678daa292d99d8b3da 2024-11-16T10:25:24,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/b572394d9b614a11a0009785bff46183 is 50, key is test_row_1/B:col10/1731752721653/Put/seqid=0 2024-11-16T10:25:24,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752784077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:24,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752784077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:24,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752784077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:24,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:24,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752784096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:24,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:24,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752784096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:24,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:24,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742144_1320 (size=9657) 2024-11-16T10:25:24,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:24,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752784287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:24,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:24,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752784287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:24,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:24,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:24,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752784287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:24,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:24,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:24,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752784306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:24,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:24,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:24,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752784306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:24,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:24,521 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/b572394d9b614a11a0009785bff46183 2024-11-16T10:25:24,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/0ddc825308654760a3cfd9fdee9b1d77 is 50, key is test_row_1/C:col10/1731752721653/Put/seqid=0 2024-11-16T10:25:24,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742145_1321 (size=9657) 2024-11-16T10:25:24,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752784596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:24,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:24,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752784607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:24,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:24,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752784607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:24,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:24,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:24,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752784616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752784616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:24,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:24,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:24,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:24,978 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/0ddc825308654760a3cfd9fdee9b1d77 2024-11-16T10:25:24,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/4a7764e881914c678daa292d99d8b3da as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/4a7764e881914c678daa292d99d8b3da 2024-11-16T10:25:24,989 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/4a7764e881914c678daa292d99d8b3da, entries=100, sequenceid=88, filesize=21.8 K 2024-11-16T10:25:24,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/b572394d9b614a11a0009785bff46183 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/b572394d9b614a11a0009785bff46183 2024-11-16T10:25:24,997 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/b572394d9b614a11a0009785bff46183, entries=100, sequenceid=88, filesize=9.4 K 2024-11-16T10:25:24,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/0ddc825308654760a3cfd9fdee9b1d77 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/0ddc825308654760a3cfd9fdee9b1d77 2024-11-16T10:25:25,003 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/0ddc825308654760a3cfd9fdee9b1d77, entries=100, sequenceid=88, filesize=9.4 K 2024-11-16T10:25:25,004 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for a591075eabfc58d555a22f619c4575e6 in 1418ms, sequenceid=88, compaction requested=false 2024-11-16T10:25:25,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:25,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:25,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-16T10:25:25,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-16T10:25:25,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-16T10:25:25,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4260 sec 2024-11-16T10:25:25,012 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 3.4330 sec 2024-11-16T10:25:25,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:25,123 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:25:25,123 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a591075eabfc58d555a22f619c4575e6 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-16T10:25:25,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111619422f5a40fe4285aab260be53ac3b58_a591075eabfc58d555a22f619c4575e6 is 50, key is test_row_0/A:col10/1731752723945/Put/seqid=0 2024-11-16T10:25:25,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742146_1322 (size=14594) 2024-11-16T10:25:25,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:25,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:25,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752785136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:25,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752785137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:25,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:25,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752785138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:25,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:25,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:25,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752785147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:25,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:25,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:25,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752785148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:25,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:25,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:25,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752785257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:25,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:25,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752785257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:25,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:25,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:25,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752785266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:25,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:25,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:25,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752785266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:25,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:25,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:25,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752785466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:25,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:25,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752785477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:25,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:25,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:25,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752785486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:25,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:25,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752785486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:25,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:25,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:25,546 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:25,550 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111619422f5a40fe4285aab260be53ac3b58_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111619422f5a40fe4285aab260be53ac3b58_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:25,551 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/9e0542df85584031ba2e83115c67f69b, store: [table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:25,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/9e0542df85584031ba2e83115c67f69b is 175, key is test_row_0/A:col10/1731752723945/Put/seqid=0 2024-11-16T10:25:25,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742147_1323 (size=39549) 2024-11-16T10:25:25,558 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/9e0542df85584031ba2e83115c67f69b 2024-11-16T10:25:25,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/4427a74a93924a49b9ed68e8c300f54c is 50, key is test_row_0/B:col10/1731752723945/Put/seqid=0 2024-11-16T10:25:25,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742148_1324 (size=12001) 2024-11-16T10:25:25,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T10:25:25,746 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:25:25,746 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:25:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-16T10:25:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T10:25:25,748 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:25:25,749 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:25:25,749 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:25:25,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:25,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752785776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:25,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:25,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752785796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:25,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:25,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:25,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752785807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:25,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:25,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752785807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:25,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:25,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:25,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:25,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T10:25:25,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T10:25:25,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:25,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:25,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:25,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:25,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:25,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:26,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/4427a74a93924a49b9ed68e8c300f54c 2024-11-16T10:25:26,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/5dbd2a08dcbd44d48b68bace0a243841 is 50, key is test_row_0/C:col10/1731752723945/Put/seqid=0 2024-11-16T10:25:26,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742149_1325 (size=12001) 2024-11-16T10:25:26,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/5dbd2a08dcbd44d48b68bace0a243841 2024-11-16T10:25:26,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/9e0542df85584031ba2e83115c67f69b as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/9e0542df85584031ba2e83115c67f69b 2024-11-16T10:25:26,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T10:25:26,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:26,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:26,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:26,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/9e0542df85584031ba2e83115c67f69b, entries=200, sequenceid=119, filesize=38.6 K 2024-11-16T10:25:26,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:26,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:26,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:26,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/4427a74a93924a49b9ed68e8c300f54c as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/4427a74a93924a49b9ed68e8c300f54c 2024-11-16T10:25:26,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T10:25:26,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/4427a74a93924a49b9ed68e8c300f54c, entries=150, sequenceid=119, filesize=11.7 K 2024-11-16T10:25:26,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/5dbd2a08dcbd44d48b68bace0a243841 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/5dbd2a08dcbd44d48b68bace0a243841 2024-11-16T10:25:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/5dbd2a08dcbd44d48b68bace0a243841, entries=150, sequenceid=119, filesize=11.7 K 2024-11-16T10:25:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for a591075eabfc58d555a22f619c4575e6 in 955ms, sequenceid=119, compaction requested=true 2024-11-16T10:25:26,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:26,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a591075eabfc58d555a22f619c4575e6:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:25:26,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:26,079 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:25:26,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a591075eabfc58d555a22f619c4575e6:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:25:26,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a591075eabfc58d555a22f619c4575e6:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:25:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T10:25:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,088 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:25:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,089 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:25:26,089 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): a591075eabfc58d555a22f619c4575e6/A is initiating minor compaction (all files) 2024-11-16T10:25:26,089 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a591075eabfc58d555a22f619c4575e6/A in TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,089 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/0b5c3d8dd19e43cab2eeeacd4bfc463e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/4a7764e881914c678daa292d99d8b3da, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/9e0542df85584031ba2e83115c67f69b] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp, totalSize=90.8 K 2024-11-16T10:25:26,089 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:26,089 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. files: [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/0b5c3d8dd19e43cab2eeeacd4bfc463e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/4a7764e881914c678daa292d99d8b3da, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/9e0542df85584031ba2e83115c67f69b] 2024-11-16T10:25:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,090 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:25:26,090 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): a591075eabfc58d555a22f619c4575e6/B is initiating minor compaction (all files) 2024-11-16T10:25:26,090 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a591075eabfc58d555a22f619c4575e6/B in TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,090 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/0f36fed28754404390572fa491d35756, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/b572394d9b614a11a0009785bff46183, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/4427a74a93924a49b9ed68e8c300f54c] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp, totalSize=33.0 K 2024-11-16T10:25:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,090 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b5c3d8dd19e43cab2eeeacd4bfc463e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731752720934 2024-11-16T10:25:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,091 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f36fed28754404390572fa491d35756, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731752720934 2024-11-16T10:25:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,091 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a7764e881914c678daa292d99d8b3da, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731752721644 2024-11-16T10:25:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,091 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting b572394d9b614a11a0009785bff46183, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731752721644 2024-11-16T10:25:26,091 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 4427a74a93924a49b9ed68e8c300f54c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731752723945 2024-11-16T10:25:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,091 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e0542df85584031ba2e83115c67f69b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731752723945 2024-11-16T10:25:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,113 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,126 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a591075eabfc58d555a22f619c4575e6#B#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:26,126 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/b974048b9b90443bb779258b85891b92 is 50, key is test_row_0/B:col10/1731752723945/Put/seqid=0 2024-11-16T10:25:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,137 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116e515c5e6756a4b48bb09ccbc198967a6_a591075eabfc58d555a22f619c4575e6 store=[table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:26,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,139 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116e515c5e6756a4b48bb09ccbc198967a6_a591075eabfc58d555a22f619c4575e6, store=[table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:26,139 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e515c5e6756a4b48bb09ccbc198967a6_a591075eabfc58d555a22f619c4575e6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742150_1326 (size=12241) 2024-11-16T10:25:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742151_1327 (size=4469) 2024-11-16T10:25:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,194 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a591075eabfc58d555a22f619c4575e6#A#compaction#278 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,194 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/48f685f2bf1d4ea7b8c36fbcad5613a0 is 175, key is test_row_0/A:col10/1731752723945/Put/seqid=0 2024-11-16T10:25:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T10:25:26,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,215 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing a591075eabfc58d555a22f619c4575e6 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-16T10:25:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742152_1328 (size=31195) 2024-11-16T10:25:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,244 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/48f685f2bf1d4ea7b8c36fbcad5613a0 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/48f685f2bf1d4ea7b8c36fbcad5613a0 2024-11-16T10:25:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,251 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a591075eabfc58d555a22f619c4575e6/A of a591075eabfc58d555a22f619c4575e6 into 48f685f2bf1d4ea7b8c36fbcad5613a0(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:26,251 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:26,251 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., storeName=a591075eabfc58d555a22f619c4575e6/A, priority=13, startTime=1731752726078; duration=0sec 2024-11-16T10:25:26,252 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:26,252 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a591075eabfc58d555a22f619c4575e6:A 2024-11-16T10:25:26,252 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:25:26,253 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:25:26,254 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): a591075eabfc58d555a22f619c4575e6/C is initiating minor compaction (all files) 2024-11-16T10:25:26,254 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a591075eabfc58d555a22f619c4575e6/C in TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:26,254 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/80542add2d0f45deb845a1c6194350f4, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/0ddc825308654760a3cfd9fdee9b1d77, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/5dbd2a08dcbd44d48b68bace0a243841] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp, totalSize=33.0 K 2024-11-16T10:25:26,254 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80542add2d0f45deb845a1c6194350f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731752720934 2024-11-16T10:25:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,254 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ddc825308654760a3cfd9fdee9b1d77, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731752721644 2024-11-16T10:25:26,255 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5dbd2a08dcbd44d48b68bace0a243841, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731752723945 2024-11-16T10:25:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111643e191a40d8c4333ac0f1f75b18b0060_a591075eabfc58d555a22f619c4575e6 is 50, key is test_row_1/A:col10/1731752726207/Put/seqid=0 2024-11-16T10:25:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,274 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a591075eabfc58d555a22f619c4575e6#C#compaction#281 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,274 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/6f226f2739834c2bb1b931635d206c73 is 50, key is test_row_0/C:col10/1731752723945/Put/seqid=0 2024-11-16T10:25:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742153_1329 (size=12204) 2024-11-16T10:25:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,333 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111643e191a40d8c4333ac0f1f75b18b0060_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111643e191a40d8c4333ac0f1f75b18b0060_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/e9ebc91377694adaafe686ef70a1fef0, store: [table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:26,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/e9ebc91377694adaafe686ef70a1fef0 is 175, key is test_row_1/A:col10/1731752726207/Put/seqid=0 2024-11-16T10:25:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742154_1330 (size=12241) 2024-11-16T10:25:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T10:25:26,400 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/6f226f2739834c2bb1b931635d206c73 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/6f226f2739834c2bb1b931635d206c73 2024-11-16T10:25:26,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742155_1331 (size=31001) 2024-11-16T10:25:26,408 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a591075eabfc58d555a22f619c4575e6/C of a591075eabfc58d555a22f619c4575e6 into 6f226f2739834c2bb1b931635d206c73(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:26,408 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:26,408 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., storeName=a591075eabfc58d555a22f619c4575e6/C, priority=13, startTime=1731752726079; duration=0sec 2024-11-16T10:25:26,408 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:26,408 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a591075eabfc58d555a22f619c4575e6:C 2024-11-16T10:25:26,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752786464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:26,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752786465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:26,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:26,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:26,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752786465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:26,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:26,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:26,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752786468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:26,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:26,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752786480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:26,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:26,585 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/b974048b9b90443bb779258b85891b92 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/b974048b9b90443bb779258b85891b92 2024-11-16T10:25:26,591 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a591075eabfc58d555a22f619c4575e6/B of a591075eabfc58d555a22f619c4575e6 into b974048b9b90443bb779258b85891b92(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:26,591 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:26,591 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., storeName=a591075eabfc58d555a22f619c4575e6/B, priority=13, startTime=1731752726079; duration=0sec 2024-11-16T10:25:26,591 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:26,591 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a591075eabfc58d555a22f619c4575e6:B 2024-11-16T10:25:26,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752786586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:26,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:26,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752786586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:26,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:26,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752786586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:26,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752786586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:26,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752786586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:26,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:26,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:26,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:26,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:26,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752786797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:26,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:26,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:26,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752786797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:26,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752786797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:26,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:26,808 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/e9ebc91377694adaafe686ef70a1fef0 2024-11-16T10:25:26,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752786806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:26,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752786806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:26,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:26,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:26,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:26,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/b204e83bea7942fda9dcc86d76dcb92a is 50, key is test_row_1/B:col10/1731752726207/Put/seqid=0 2024-11-16T10:25:26,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742156_1332 (size=9707) 2024-11-16T10:25:26,837 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/b204e83bea7942fda9dcc86d76dcb92a 2024-11-16T10:25:26,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/4f3a17450f454857b94d1e1a14127d12 is 50, key is test_row_1/C:col10/1731752726207/Put/seqid=0 2024-11-16T10:25:26,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742157_1333 (size=9707) 2024-11-16T10:25:26,868 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/4f3a17450f454857b94d1e1a14127d12 2024-11-16T10:25:26,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/e9ebc91377694adaafe686ef70a1fef0 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/e9ebc91377694adaafe686ef70a1fef0 2024-11-16T10:25:26,886 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/e9ebc91377694adaafe686ef70a1fef0, entries=150, sequenceid=129, filesize=30.3 K 2024-11-16T10:25:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T10:25:26,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/b204e83bea7942fda9dcc86d76dcb92a as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/b204e83bea7942fda9dcc86d76dcb92a 2024-11-16T10:25:26,897 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/b204e83bea7942fda9dcc86d76dcb92a, entries=100, sequenceid=129, filesize=9.5 K 2024-11-16T10:25:26,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/4f3a17450f454857b94d1e1a14127d12 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/4f3a17450f454857b94d1e1a14127d12 2024-11-16T10:25:26,907 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/4f3a17450f454857b94d1e1a14127d12, entries=100, sequenceid=129, filesize=9.5 K 2024-11-16T10:25:26,912 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for a591075eabfc58d555a22f619c4575e6 in 697ms, sequenceid=129, compaction requested=false 2024-11-16T10:25:26,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:26,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:26,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-16T10:25:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-16T10:25:26,918 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-16T10:25:26,918 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1650 sec 2024-11-16T10:25:26,921 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.1730 sec 2024-11-16T10:25:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:27,113 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:25:27,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a591075eabfc58d555a22f619c4575e6 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-16T10:25:27,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116de8bf68b5b714ad29fe85be90d22af68_a591075eabfc58d555a22f619c4575e6 is 50, key is test_row_0/A:col10/1731752726461/Put/seqid=0 2024-11-16T10:25:27,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742158_1334 (size=14794) 2024-11-16T10:25:27,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:27,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752787127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752787127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:27,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752787133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:27,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:27,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:27,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:27,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752787136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752787137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:27,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752787246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:27,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752787256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:27,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:27,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752787257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:27,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:27,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:27,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752787456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:27,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:27,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752787469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:27,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:27,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752787476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:27,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:27,529 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:27,533 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116de8bf68b5b714ad29fe85be90d22af68_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116de8bf68b5b714ad29fe85be90d22af68_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:27,533 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/a7a303b46b4e49c4b427d5373b0fffee, store: [table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:27,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/a7a303b46b4e49c4b427d5373b0fffee is 175, key is test_row_0/A:col10/1731752726461/Put/seqid=0 2024-11-16T10:25:27,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742159_1335 (size=39749) 2024-11-16T10:25:27,551 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/a7a303b46b4e49c4b427d5373b0fffee 2024-11-16T10:25:27,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/eb5914493fad45caae5a38378734e9dc is 50, key is test_row_0/B:col10/1731752726461/Put/seqid=0 2024-11-16T10:25:27,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742160_1336 (size=12151) 2024-11-16T10:25:27,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/eb5914493fad45caae5a38378734e9dc 2024-11-16T10:25:27,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/99114140026e4067a1eeb8203e16bf52 is 50, key is test_row_0/C:col10/1731752726461/Put/seqid=0 2024-11-16T10:25:27,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742161_1337 (size=12151) 2024-11-16T10:25:27,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/99114140026e4067a1eeb8203e16bf52 2024-11-16T10:25:27,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/a7a303b46b4e49c4b427d5373b0fffee as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/a7a303b46b4e49c4b427d5373b0fffee 2024-11-16T10:25:27,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752787646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:27,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:27,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/a7a303b46b4e49c4b427d5373b0fffee, entries=200, sequenceid=159, filesize=38.8 K 2024-11-16T10:25:27,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/eb5914493fad45caae5a38378734e9dc as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/eb5914493fad45caae5a38378734e9dc 2024-11-16T10:25:27,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/eb5914493fad45caae5a38378734e9dc, entries=150, sequenceid=159, filesize=11.9 K 2024-11-16T10:25:27,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752787656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:27,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:27,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/99114140026e4067a1eeb8203e16bf52 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/99114140026e4067a1eeb8203e16bf52 2024-11-16T10:25:27,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/99114140026e4067a1eeb8203e16bf52, entries=150, sequenceid=159, filesize=11.9 K 2024-11-16T10:25:27,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for a591075eabfc58d555a22f619c4575e6 in 558ms, sequenceid=159, compaction requested=true 2024-11-16T10:25:27,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:27,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a591075eabfc58d555a22f619c4575e6:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:25:27,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:27,672 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:25:27,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a591075eabfc58d555a22f619c4575e6:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:25:27,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:27,672 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:25:27,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a591075eabfc58d555a22f619c4575e6:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:25:27,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:27,673 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101945 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:25:27,673 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:25:27,673 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): a591075eabfc58d555a22f619c4575e6/B is initiating minor compaction (all files) 2024-11-16T10:25:27,673 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): a591075eabfc58d555a22f619c4575e6/A is initiating minor compaction (all files) 2024-11-16T10:25:27,673 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a591075eabfc58d555a22f619c4575e6/A in TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:27,673 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a591075eabfc58d555a22f619c4575e6/B in TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:27,673 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/b974048b9b90443bb779258b85891b92, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/b204e83bea7942fda9dcc86d76dcb92a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/eb5914493fad45caae5a38378734e9dc] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp, totalSize=33.3 K 2024-11-16T10:25:27,673 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/48f685f2bf1d4ea7b8c36fbcad5613a0, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/e9ebc91377694adaafe686ef70a1fef0, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/a7a303b46b4e49c4b427d5373b0fffee] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp, totalSize=99.6 K 2024-11-16T10:25:27,673 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:27,673 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. files: [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/48f685f2bf1d4ea7b8c36fbcad5613a0, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/e9ebc91377694adaafe686ef70a1fef0, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/a7a303b46b4e49c4b427d5373b0fffee] 2024-11-16T10:25:27,674 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting b974048b9b90443bb779258b85891b92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731752723945 2024-11-16T10:25:27,674 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48f685f2bf1d4ea7b8c36fbcad5613a0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731752723945 2024-11-16T10:25:27,674 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting b204e83bea7942fda9dcc86d76dcb92a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731752725145 2024-11-16T10:25:27,675 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9ebc91377694adaafe686ef70a1fef0, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731752725145 2024-11-16T10:25:27,675 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting eb5914493fad45caae5a38378734e9dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731752726461 2024-11-16T10:25:27,675 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7a303b46b4e49c4b427d5373b0fffee, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731752726461 2024-11-16T10:25:27,699 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a591075eabfc58d555a22f619c4575e6#B#compaction#287 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:27,699 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/23a829e35c92458d906207cd582ea200 is 50, key is test_row_0/B:col10/1731752726461/Put/seqid=0 2024-11-16T10:25:27,701 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:27,705 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116ae28c9df80db4e4197a44685d1b43df9_a591075eabfc58d555a22f619c4575e6 store=[table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:27,707 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116ae28c9df80db4e4197a44685d1b43df9_a591075eabfc58d555a22f619c4575e6, store=[table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:27,707 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ae28c9df80db4e4197a44685d1b43df9_a591075eabfc58d555a22f619c4575e6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:27,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742163_1339 (size=4469) 2024-11-16T10:25:27,750 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a591075eabfc58d555a22f619c4575e6#A#compaction#288 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:27,753 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/95f58abf786147908bb63f35c444034c is 175, key is test_row_0/A:col10/1731752726461/Put/seqid=0 2024-11-16T10:25:27,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742162_1338 (size=12493) 2024-11-16T10:25:27,769 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/23a829e35c92458d906207cd582ea200 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/23a829e35c92458d906207cd582ea200 2024-11-16T10:25:27,775 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a591075eabfc58d555a22f619c4575e6/B of a591075eabfc58d555a22f619c4575e6 into 23a829e35c92458d906207cd582ea200(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:27,775 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:27,775 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., storeName=a591075eabfc58d555a22f619c4575e6/B, priority=13, startTime=1731752727672; duration=0sec 2024-11-16T10:25:27,775 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:27,775 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a591075eabfc58d555a22f619c4575e6:B 2024-11-16T10:25:27,775 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:25:27,777 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:25:27,777 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): a591075eabfc58d555a22f619c4575e6/C is initiating minor compaction (all files) 2024-11-16T10:25:27,777 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a591075eabfc58d555a22f619c4575e6/C in TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:27,777 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/6f226f2739834c2bb1b931635d206c73, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/4f3a17450f454857b94d1e1a14127d12, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/99114140026e4067a1eeb8203e16bf52] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp, totalSize=33.3 K 2024-11-16T10:25:27,778 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f226f2739834c2bb1b931635d206c73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731752723945 2024-11-16T10:25:27,778 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f3a17450f454857b94d1e1a14127d12, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731752725145 2024-11-16T10:25:27,779 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 99114140026e4067a1eeb8203e16bf52, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731752726461 2024-11-16T10:25:27,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742164_1340 (size=31447) 2024-11-16T10:25:27,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:27,792 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:25:27,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a591075eabfc58d555a22f619c4575e6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T10:25:27,808 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a591075eabfc58d555a22f619c4575e6#C#compaction#289 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:27,808 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/288c39efec77423b86aa0e5def9d44e6 is 50, key is test_row_0/C:col10/1731752726461/Put/seqid=0 2024-11-16T10:25:27,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d11f9b40708c4d2ab18d598a85f1793e_a591075eabfc58d555a22f619c4575e6 is 50, key is test_row_0/A:col10/1731752727784/Put/seqid=0 2024-11-16T10:25:27,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742165_1341 (size=12493) 2024-11-16T10:25:27,873 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/288c39efec77423b86aa0e5def9d44e6 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/288c39efec77423b86aa0e5def9d44e6 2024-11-16T10:25:27,879 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a591075eabfc58d555a22f619c4575e6/C of a591075eabfc58d555a22f619c4575e6 into 288c39efec77423b86aa0e5def9d44e6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:27,879 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:27,879 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., storeName=a591075eabfc58d555a22f619c4575e6/C, priority=13, startTime=1731752727672; duration=0sec 2024-11-16T10:25:27,880 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:27,880 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a591075eabfc58d555a22f619c4575e6:C 2024-11-16T10:25:27,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742166_1342 (size=14794) 2024-11-16T10:25:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T10:25:27,906 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:25:27,907 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:25:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-16T10:25:27,909 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:25:27,910 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:25:27,910 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:25:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T10:25:27,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752787906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:27,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:27,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752787913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:27,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:27,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:27,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752787919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:27,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:27,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:28,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T10:25:28,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752788026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:28,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752788027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:28,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:28,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:28,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:28,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:28,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:28,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:28,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752788036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:28,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:28,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:28,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:28,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T10:25:28,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:28,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:28,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:28,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:28,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:28,187 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/95f58abf786147908bb63f35c444034c as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/95f58abf786147908bb63f35c444034c 2024-11-16T10:25:28,193 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a591075eabfc58d555a22f619c4575e6/A of a591075eabfc58d555a22f619c4575e6 into 95f58abf786147908bb63f35c444034c(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:28,193 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:28,193 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., storeName=a591075eabfc58d555a22f619c4575e6/A, priority=13, startTime=1731752727672; duration=0sec 2024-11-16T10:25:28,193 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:28,193 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a591075eabfc58d555a22f619c4575e6:A 2024-11-16T10:25:28,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T10:25:28,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:28,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:28,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:28,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:28,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:28,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:28,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T10:25:28,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:28,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752788236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:28,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:28,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:28,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:28,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:28,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752788246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:28,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:28,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:28,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:28,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752788256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:28,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:28,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:28,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:28,301 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:28,310 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d11f9b40708c4d2ab18d598a85f1793e_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d11f9b40708c4d2ab18d598a85f1793e_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:28,311 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/1ee34b8bb37c43eb8e8be30ab5f88293, store: [table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:28,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/1ee34b8bb37c43eb8e8be30ab5f88293 is 175, key is test_row_0/A:col10/1731752727784/Put/seqid=0 2024-11-16T10:25:28,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742167_1343 (size=39749) 2024-11-16T10:25:28,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T10:25:28,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:28,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:28,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:28,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:28,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:28,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T10:25:28,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:28,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:28,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:28,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:28,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T10:25:28,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752788546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:28,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:28,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:28,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:28,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752788557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:28,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:28,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:28,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:28,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752788566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:28,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:28,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:28,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:28,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752788667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:28,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752788667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:28,675 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_2' in a591075eabfc58d555a22f619c4575e6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:25:28,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:28,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:28,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:28,676 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_0' in a591075eabfc58d555a22f619c4575e6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:25:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:28,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T10:25:28,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:28,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:28,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:28,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:28,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:28,740 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/1ee34b8bb37c43eb8e8be30ab5f88293 2024-11-16T10:25:28,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/248710cd6ebd41a996f090d44a3e6f6f is 50, key is test_row_0/B:col10/1731752727784/Put/seqid=0 2024-11-16T10:25:28,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742168_1344 (size=12151) 2024-11-16T10:25:28,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/248710cd6ebd41a996f090d44a3e6f6f 2024-11-16T10:25:28,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/039dfcb6b5174bef933adec064e8607b is 50, key is test_row_0/C:col10/1731752727784/Put/seqid=0 2024-11-16T10:25:28,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T10:25:28,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:28,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:28,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:28,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:28,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:28,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742169_1345 (size=12151) 2024-11-16T10:25:28,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T10:25:28,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:28,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:28,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:28,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:28,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T10:25:29,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752789066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:29,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752789066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:29,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752789087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:29,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:29,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:29,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:29,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T10:25:29,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:29,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:29,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:29,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:29,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:29,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/039dfcb6b5174bef933adec064e8607b 2024-11-16T10:25:29,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/1ee34b8bb37c43eb8e8be30ab5f88293 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/1ee34b8bb37c43eb8e8be30ab5f88293 2024-11-16T10:25:29,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/1ee34b8bb37c43eb8e8be30ab5f88293, entries=200, sequenceid=172, filesize=38.8 K 2024-11-16T10:25:29,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/248710cd6ebd41a996f090d44a3e6f6f as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/248710cd6ebd41a996f090d44a3e6f6f 2024-11-16T10:25:29,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/248710cd6ebd41a996f090d44a3e6f6f, entries=150, sequenceid=172, filesize=11.9 K 2024-11-16T10:25:29,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/039dfcb6b5174bef933adec064e8607b as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/039dfcb6b5174bef933adec064e8607b 2024-11-16T10:25:29,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/039dfcb6b5174bef933adec064e8607b, entries=150, sequenceid=172, filesize=11.9 K 2024-11-16T10:25:29,280 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for a591075eabfc58d555a22f619c4575e6 in 1488ms, sequenceid=172, compaction requested=false 2024-11-16T10:25:29,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:29,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T10:25:29,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:29,308 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing a591075eabfc58d555a22f619c4575e6 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T10:25:29,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165bd2a810c6b948db881de81b13cfca73_a591075eabfc58d555a22f619c4575e6 is 50, key is test_row_0/A:col10/1731752727917/Put/seqid=0 2024-11-16T10:25:29,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742170_1346 (size=12304) 2024-11-16T10:25:29,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:29,763 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165bd2a810c6b948db881de81b13cfca73_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165bd2a810c6b948db881de81b13cfca73_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:29,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/65ab26bd7f0242e888754d62b2daea2d, store: [table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:29,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/65ab26bd7f0242e888754d62b2daea2d is 175, key is test_row_0/A:col10/1731752727917/Put/seqid=0 2024-11-16T10:25:29,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742171_1347 (size=31105) 2024-11-16T10:25:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T10:25:30,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:30,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:30,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:30,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752790124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:30,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:30,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:30,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752790124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:30,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:30,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752790125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:30,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:30,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:30,188 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/65ab26bd7f0242e888754d62b2daea2d 2024-11-16T10:25:30,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/afb0865d11a44995b0869d8513f90924 is 50, key is test_row_0/B:col10/1731752727917/Put/seqid=0 2024-11-16T10:25:30,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742172_1348 (size=12151) 2024-11-16T10:25:30,226 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/afb0865d11a44995b0869d8513f90924 2024-11-16T10:25:30,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/6054268fb8cd4525a00858de8b18630b is 50, key is test_row_0/C:col10/1731752727917/Put/seqid=0 2024-11-16T10:25:30,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742173_1349 (size=12151) 2024-11-16T10:25:30,240 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/6054268fb8cd4525a00858de8b18630b 2024-11-16T10:25:30,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:30,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752790236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:30,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:30,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/65ab26bd7f0242e888754d62b2daea2d as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/65ab26bd7f0242e888754d62b2daea2d 2024-11-16T10:25:30,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752790248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:30,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752790248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:30,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:30,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:30,258 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/65ab26bd7f0242e888754d62b2daea2d, entries=150, sequenceid=198, filesize=30.4 K 2024-11-16T10:25:30,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/afb0865d11a44995b0869d8513f90924 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/afb0865d11a44995b0869d8513f90924 2024-11-16T10:25:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,265 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/afb0865d11a44995b0869d8513f90924, entries=150, sequenceid=198, filesize=11.9 K 2024-11-16T10:25:30,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/6054268fb8cd4525a00858de8b18630b as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/6054268fb8cd4525a00858de8b18630b 2024-11-16T10:25:30,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,270 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/6054268fb8cd4525a00858de8b18630b, entries=150, sequenceid=198, filesize=11.9 K 2024-11-16T10:25:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,271 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for a591075eabfc58d555a22f619c4575e6 in 963ms, sequenceid=198, compaction requested=true 2024-11-16T10:25:30,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:30,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:30,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-16T10:25:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-16T10:25:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,276 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-16T10:25:30,276 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3640 sec 2024-11-16T10:25:30,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,278 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.3700 sec 2024-11-16T10:25:30,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:30,482 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:25:30,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a591075eabfc58d555a22f619c4575e6 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-16T10:25:30,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fcbde42a565748bfbd13e3c95d22122e_a591075eabfc58d555a22f619c4575e6 is 50, key is test_row_0/A:col10/1731752730121/Put/seqid=0 2024-11-16T10:25:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742174_1350 (size=12304) 2024-11-16T10:25:30,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,545 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:30,555 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fcbde42a565748bfbd13e3c95d22122e_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fcbde42a565748bfbd13e3c95d22122e_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:30,561 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/54e0cac28c144af2861f348aa2ebef21, store: [table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:30,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/54e0cac28c144af2861f348aa2ebef21 is 175, key is test_row_0/A:col10/1731752730121/Put/seqid=0 2024-11-16T10:25:30,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742175_1351 (size=31101) 2024-11-16T10:25:30,578 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/54e0cac28c144af2861f348aa2ebef21 2024-11-16T10:25:30,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/a7264bc598474e42b4b82f1471e5eb4c is 50, key is test_row_0/B:col10/1731752730121/Put/seqid=0 2024-11-16T10:25:30,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742176_1352 (size=9757) 2024-11-16T10:25:30,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/a7264bc598474e42b4b82f1471e5eb4c 2024-11-16T10:25:30,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:30,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752790601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:30,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/9d98439aeb904d259f45ac1e269785cc is 50, key is test_row_0/C:col10/1731752730121/Put/seqid=0 2024-11-16T10:25:30,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:30,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752790610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:30,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:30,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742177_1353 (size=9757) 2024-11-16T10:25:30,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:30,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752790614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:30,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:30,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/9d98439aeb904d259f45ac1e269785cc 2024-11-16T10:25:30,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/54e0cac28c144af2861f348aa2ebef21 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/54e0cac28c144af2861f348aa2ebef21 2024-11-16T10:25:30,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/54e0cac28c144af2861f348aa2ebef21, entries=150, sequenceid=212, filesize=30.4 K 2024-11-16T10:25:30,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/a7264bc598474e42b4b82f1471e5eb4c as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/a7264bc598474e42b4b82f1471e5eb4c 2024-11-16T10:25:30,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/a7264bc598474e42b4b82f1471e5eb4c, entries=100, sequenceid=212, filesize=9.5 K 2024-11-16T10:25:30,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/9d98439aeb904d259f45ac1e269785cc as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/9d98439aeb904d259f45ac1e269785cc 2024-11-16T10:25:30,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/9d98439aeb904d259f45ac1e269785cc, entries=100, sequenceid=212, filesize=9.5 K 2024-11-16T10:25:30,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for a591075eabfc58d555a22f619c4575e6 in 177ms, sequenceid=212, compaction requested=true 2024-11-16T10:25:30,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:30,660 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:25:30,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a591075eabfc58d555a22f619c4575e6:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:25:30,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:30,660 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:25:30,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a591075eabfc58d555a22f619c4575e6:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:25:30,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:30,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a591075eabfc58d555a22f619c4575e6:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:25:30,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:30,666 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133402 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:25:30,666 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): a591075eabfc58d555a22f619c4575e6/A is initiating minor compaction (all files) 2024-11-16T10:25:30,666 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a591075eabfc58d555a22f619c4575e6/A in TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:30,666 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/95f58abf786147908bb63f35c444034c, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/1ee34b8bb37c43eb8e8be30ab5f88293, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/65ab26bd7f0242e888754d62b2daea2d, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/54e0cac28c144af2861f348aa2ebef21] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp, totalSize=130.3 K 2024-11-16T10:25:30,666 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:30,666 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. files: [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/95f58abf786147908bb63f35c444034c, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/1ee34b8bb37c43eb8e8be30ab5f88293, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/65ab26bd7f0242e888754d62b2daea2d, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/54e0cac28c144af2861f348aa2ebef21] 2024-11-16T10:25:30,667 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:25:30,667 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): a591075eabfc58d555a22f619c4575e6/B is initiating minor compaction (all files) 2024-11-16T10:25:30,667 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a591075eabfc58d555a22f619c4575e6/B in TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:30,667 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/23a829e35c92458d906207cd582ea200, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/248710cd6ebd41a996f090d44a3e6f6f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/afb0865d11a44995b0869d8513f90924, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/a7264bc598474e42b4b82f1471e5eb4c] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp, totalSize=45.5 K 2024-11-16T10:25:30,667 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95f58abf786147908bb63f35c444034c, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731752726461 2024-11-16T10:25:30,668 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 23a829e35c92458d906207cd582ea200, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731752726461 2024-11-16T10:25:30,668 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ee34b8bb37c43eb8e8be30ab5f88293, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731752727122 2024-11-16T10:25:30,668 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 248710cd6ebd41a996f090d44a3e6f6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731752727771 2024-11-16T10:25:30,668 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65ab26bd7f0242e888754d62b2daea2d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731752727883 2024-11-16T10:25:30,669 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54e0cac28c144af2861f348aa2ebef21, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731752730121 2024-11-16T10:25:30,669 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting afb0865d11a44995b0869d8513f90924, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731752727883 2024-11-16T10:25:30,669 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting a7264bc598474e42b4b82f1471e5eb4c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731752730121 2024-11-16T10:25:30,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:30,695 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:25:30,695 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a591075eabfc58d555a22f619c4575e6 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T10:25:30,697 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:30,719 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a591075eabfc58d555a22f619c4575e6#B#compaction#300 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:30,720 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/4cf9dd56da90408db7720ddf7236853a is 50, key is test_row_0/B:col10/1731752730121/Put/seqid=0 2024-11-16T10:25:30,725 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411165cdaef2e8fa243f9bc59b7f9f73345d3_a591075eabfc58d555a22f619c4575e6 store=[table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:30,727 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411165cdaef2e8fa243f9bc59b7f9f73345d3_a591075eabfc58d555a22f619c4575e6, store=[table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:30,728 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165cdaef2e8fa243f9bc59b7f9f73345d3_a591075eabfc58d555a22f619c4575e6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:30,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162def8e306a1d4bb6baefef95fd31708d_a591075eabfc58d555a22f619c4575e6 is 50, key is test_row_0/A:col10/1731752730566/Put/seqid=0 2024-11-16T10:25:30,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742178_1354 (size=12629) 2024-11-16T10:25:30,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742179_1355 (size=4469) 2024-11-16T10:25:30,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742180_1356 (size=14794) 2024-11-16T10:25:30,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:30,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:30,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752790738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:30,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752790739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:30,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:30,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752790786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:30,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:30,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752790786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:30,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:30,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752790786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:30,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:30,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:30,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:30,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752790896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:30,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752790896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:30,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752790906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:30,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752790906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:30,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752790906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:31,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752791116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:31,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:31,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752791116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:31,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:31,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:31,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:31,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:31,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752791117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:31,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:31,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:31,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752791117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:31,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:31,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:31,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:31,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:31,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752791117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:31,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:31,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:31,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:31,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:31,168 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/4cf9dd56da90408db7720ddf7236853a as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/4cf9dd56da90408db7720ddf7236853a 2024-11-16T10:25:31,177 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a591075eabfc58d555a22f619c4575e6/B of a591075eabfc58d555a22f619c4575e6 into 4cf9dd56da90408db7720ddf7236853a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:31,177 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:31,177 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., storeName=a591075eabfc58d555a22f619c4575e6/B, priority=12, startTime=1731752730660; duration=0sec 2024-11-16T10:25:31,177 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:31,177 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a591075eabfc58d555a22f619c4575e6:B 2024-11-16T10:25:31,177 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:25:31,178 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:25:31,178 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): a591075eabfc58d555a22f619c4575e6/C is initiating minor compaction (all files) 2024-11-16T10:25:31,178 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a591075eabfc58d555a22f619c4575e6/C in TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:31,178 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/288c39efec77423b86aa0e5def9d44e6, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/039dfcb6b5174bef933adec064e8607b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/6054268fb8cd4525a00858de8b18630b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/9d98439aeb904d259f45ac1e269785cc] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp, totalSize=45.5 K 2024-11-16T10:25:31,178 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 288c39efec77423b86aa0e5def9d44e6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731752726461 2024-11-16T10:25:31,179 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 039dfcb6b5174bef933adec064e8607b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731752727771 2024-11-16T10:25:31,179 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 6054268fb8cd4525a00858de8b18630b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731752727883 2024-11-16T10:25:31,179 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d98439aeb904d259f45ac1e269785cc, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731752730121 2024-11-16T10:25:31,180 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:31,183 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a591075eabfc58d555a22f619c4575e6#A#compaction#299 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:31,184 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/1855c6c68c4b407f89b9edf60ffc0d6c is 175, key is test_row_0/A:col10/1731752730121/Put/seqid=0 2024-11-16T10:25:31,184 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162def8e306a1d4bb6baefef95fd31708d_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162def8e306a1d4bb6baefef95fd31708d_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:31,185 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/33662939d3374bf49cff007974669f4d, store: [table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:31,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/33662939d3374bf49cff007974669f4d is 175, key is test_row_0/A:col10/1731752730566/Put/seqid=0 2024-11-16T10:25:31,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742181_1357 (size=31690) 2024-11-16T10:25:31,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742182_1358 (size=39749) 2024-11-16T10:25:31,203 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a591075eabfc58d555a22f619c4575e6#C#compaction#302 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:31,204 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/77a919f2d28e469cbbc11e263916a3e9 is 50, key is test_row_0/C:col10/1731752730121/Put/seqid=0 2024-11-16T10:25:31,204 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/33662939d3374bf49cff007974669f4d 2024-11-16T10:25:31,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742183_1359 (size=12629) 2024-11-16T10:25:31,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/efda052e7c5c4256b651908ea1fc74c4 is 50, key is test_row_0/B:col10/1731752730566/Put/seqid=0 2024-11-16T10:25:31,221 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/77a919f2d28e469cbbc11e263916a3e9 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/77a919f2d28e469cbbc11e263916a3e9 2024-11-16T10:25:31,225 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a591075eabfc58d555a22f619c4575e6/C of a591075eabfc58d555a22f619c4575e6 into 77a919f2d28e469cbbc11e263916a3e9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:31,225 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:31,225 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., storeName=a591075eabfc58d555a22f619c4575e6/C, priority=12, startTime=1731752730664; duration=0sec 2024-11-16T10:25:31,225 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:31,225 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a591075eabfc58d555a22f619c4575e6:C 2024-11-16T10:25:31,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742184_1360 (size=12151) 2024-11-16T10:25:31,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752791436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:31,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752791436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:31,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:31,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:31,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:31,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:31,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:31,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:31,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752791436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:31,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:31,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752791436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:31,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:31,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:31,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752791436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:31,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:31,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:31,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:31,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:31,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:31,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:31,603 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/1855c6c68c4b407f89b9edf60ffc0d6c as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/1855c6c68c4b407f89b9edf60ffc0d6c 2024-11-16T10:25:31,607 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a591075eabfc58d555a22f619c4575e6/A of a591075eabfc58d555a22f619c4575e6 into 1855c6c68c4b407f89b9edf60ffc0d6c(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:31,607 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:31,607 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., storeName=a591075eabfc58d555a22f619c4575e6/A, priority=12, startTime=1731752730659; duration=0sec 2024-11-16T10:25:31,607 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:31,607 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a591075eabfc58d555a22f619c4575e6:A 2024-11-16T10:25:31,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/efda052e7c5c4256b651908ea1fc74c4 2024-11-16T10:25:31,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/37f3116f3d54482680e471da70d5f873 is 50, key is test_row_0/C:col10/1731752730566/Put/seqid=0 2024-11-16T10:25:31,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742185_1361 (size=12151) 2024-11-16T10:25:31,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/37f3116f3d54482680e471da70d5f873 2024-11-16T10:25:31,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/33662939d3374bf49cff007974669f4d as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/33662939d3374bf49cff007974669f4d 2024-11-16T10:25:31,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/33662939d3374bf49cff007974669f4d, entries=200, sequenceid=235, filesize=38.8 K 2024-11-16T10:25:31,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/efda052e7c5c4256b651908ea1fc74c4 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/efda052e7c5c4256b651908ea1fc74c4 2024-11-16T10:25:31,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/efda052e7c5c4256b651908ea1fc74c4, entries=150, sequenceid=235, filesize=11.9 K 2024-11-16T10:25:31,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/37f3116f3d54482680e471da70d5f873 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/37f3116f3d54482680e471da70d5f873 2024-11-16T10:25:31,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/37f3116f3d54482680e471da70d5f873, entries=150, sequenceid=235, filesize=11.9 K 2024-11-16T10:25:31,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for a591075eabfc58d555a22f619c4575e6 in 969ms, sequenceid=235, compaction requested=false 2024-11-16T10:25:31,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:31,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:31,948 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:25:31,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a591075eabfc58d555a22f619c4575e6 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-16T10:25:31,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c138a3d2d79049d6aa23a70e5fd3bfab_a591075eabfc58d555a22f619c4575e6 is 50, key is test_row_0/A:col10/1731752730739/Put/seqid=0 2024-11-16T10:25:31,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742186_1362 (size=17284) 2024-11-16T10:25:31,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:31,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752791985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:31,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:31,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:31,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:32,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:32,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752791994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:32,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:32,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:32,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752791995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:32,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:32,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:32,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752791995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:32,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:32,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752791995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:32,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:32,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:32,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T10:25:32,077 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:25:32,077 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:25:32,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-16T10:25:32,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T10:25:32,079 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:25:32,079 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:25:32,079 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:25:32,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:32,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752792106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:32,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:32,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752792106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:32,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:32,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:32,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:32,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752792116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:32,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:32,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752792116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:32,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752792116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T10:25:32,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T10:25:32,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:32,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:32,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:32,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:32,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:32,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:32,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:32,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752792316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:32,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:32,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752792316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:32,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:32,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:32,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:32,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752792326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:32,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:32,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752792326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:32,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752792326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:32,362 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:32,365 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c138a3d2d79049d6aa23a70e5fd3bfab_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c138a3d2d79049d6aa23a70e5fd3bfab_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:32,366 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/3e61c85967e24b819027d66b4e5caca6, store: [table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:32,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/3e61c85967e24b819027d66b4e5caca6 is 175, key is test_row_0/A:col10/1731752730739/Put/seqid=0 2024-11-16T10:25:32,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742187_1363 (size=48389) 2024-11-16T10:25:32,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T10:25:32,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:32,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:32,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:32,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:32,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:32,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:32,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T10:25:32,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T10:25:32,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:32,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:32,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:32,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:32,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:32,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:32,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:32,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752792626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:32,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:32,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752792626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:32,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:32,631 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,631 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,631 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:32,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:32,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752792636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:32,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752792636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:32,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752792636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:32,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T10:25:32,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:32,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:32,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:32,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:32,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:32,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:32,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T10:25:32,770 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/3e61c85967e24b819027d66b4e5caca6 2024-11-16T10:25:32,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/79763c1ed7af4f4aa1a6353c5ae9d00f is 50, key is test_row_0/B:col10/1731752730739/Put/seqid=0 2024-11-16T10:25:32,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742188_1364 (size=12151) 2024-11-16T10:25:32,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T10:25:32,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:32,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:32,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:32,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:32,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:32,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:32,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T10:25:32,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:32,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:32,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:32,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:32,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:33,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:33,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752793136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:33,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:33,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:33,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:33,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T10:25:33,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:33,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:33,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:33,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:33,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:33,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:33,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:33,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752793146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:33,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:33,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:33,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:33,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752793146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:33,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:33,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:33,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:33,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:33,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:33,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752793146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:33,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:33,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:33,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:33,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:33,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752793146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:33,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:33,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:33,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:33,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/79763c1ed7af4f4aa1a6353c5ae9d00f 2024-11-16T10:25:33,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/b4779f951d75483781d97ea2a47b7590 is 50, key is test_row_0/C:col10/1731752730739/Put/seqid=0 2024-11-16T10:25:33,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742189_1365 (size=12151) 2024-11-16T10:25:33,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T10:25:33,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T10:25:33,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:33,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:33,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:33,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:33,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:33,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:33,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T10:25:33,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:33,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:33,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:33,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:33,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:33,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:33,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/b4779f951d75483781d97ea2a47b7590 2024-11-16T10:25:33,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/3e61c85967e24b819027d66b4e5caca6 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/3e61c85967e24b819027d66b4e5caca6 2024-11-16T10:25:33,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/3e61c85967e24b819027d66b4e5caca6, entries=250, sequenceid=253, filesize=47.3 K 2024-11-16T10:25:33,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/79763c1ed7af4f4aa1a6353c5ae9d00f as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/79763c1ed7af4f4aa1a6353c5ae9d00f 2024-11-16T10:25:33,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T10:25:33,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:33,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:33,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:33,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/79763c1ed7af4f4aa1a6353c5ae9d00f, entries=150, sequenceid=253, filesize=11.9 K 2024-11-16T10:25:33,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:33,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:33,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/b4779f951d75483781d97ea2a47b7590 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/b4779f951d75483781d97ea2a47b7590 2024-11-16T10:25:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:33,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/b4779f951d75483781d97ea2a47b7590, entries=150, sequenceid=253, filesize=11.9 K 2024-11-16T10:25:33,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for a591075eabfc58d555a22f619c4575e6 in 1663ms, sequenceid=253, compaction requested=true 2024-11-16T10:25:33,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:33,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a591075eabfc58d555a22f619c4575e6:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:25:33,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:33,611 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:25:33,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a591075eabfc58d555a22f619c4575e6:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:25:33,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:33,611 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:25:33,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a591075eabfc58d555a22f619c4575e6:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:25:33,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:33,612 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:25:33,612 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119828 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:25:33,612 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): a591075eabfc58d555a22f619c4575e6/A is initiating minor compaction (all files) 2024-11-16T10:25:33,612 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): a591075eabfc58d555a22f619c4575e6/B is initiating minor compaction (all files) 2024-11-16T10:25:33,612 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a591075eabfc58d555a22f619c4575e6/B in TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:33,612 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a591075eabfc58d555a22f619c4575e6/A in TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:33,613 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/4cf9dd56da90408db7720ddf7236853a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/efda052e7c5c4256b651908ea1fc74c4, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/79763c1ed7af4f4aa1a6353c5ae9d00f] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp, totalSize=36.1 K 2024-11-16T10:25:33,613 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/1855c6c68c4b407f89b9edf60ffc0d6c, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/33662939d3374bf49cff007974669f4d, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/3e61c85967e24b819027d66b4e5caca6] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp, totalSize=117.0 K 2024-11-16T10:25:33,613 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:33,613 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. files: [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/1855c6c68c4b407f89b9edf60ffc0d6c, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/33662939d3374bf49cff007974669f4d, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/3e61c85967e24b819027d66b4e5caca6] 2024-11-16T10:25:33,613 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cf9dd56da90408db7720ddf7236853a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731752727908 2024-11-16T10:25:33,613 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1855c6c68c4b407f89b9edf60ffc0d6c, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731752727908 2024-11-16T10:25:33,613 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33662939d3374bf49cff007974669f4d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731752730566 2024-11-16T10:25:33,613 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting efda052e7c5c4256b651908ea1fc74c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731752730566 2024-11-16T10:25:33,613 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e61c85967e24b819027d66b4e5caca6, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731752730739 2024-11-16T10:25:33,613 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 79763c1ed7af4f4aa1a6353c5ae9d00f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731752730739 2024-11-16T10:25:33,619 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:33,621 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a591075eabfc58d555a22f619c4575e6#B#compaction#308 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:33,621 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/ea6ebb466e824dc9bd02d63510e76195 is 50, key is test_row_0/B:col10/1731752730739/Put/seqid=0 2024-11-16T10:25:33,623 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116b89e979a21ed4f07bda942eaeea2face_a591075eabfc58d555a22f619c4575e6 store=[table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:33,625 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116b89e979a21ed4f07bda942eaeea2face_a591075eabfc58d555a22f619c4575e6, store=[table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:33,625 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b89e979a21ed4f07bda942eaeea2face_a591075eabfc58d555a22f619c4575e6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:33,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742190_1366 (size=12731) 2024-11-16T10:25:33,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742191_1367 (size=4469) 2024-11-16T10:25:33,641 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a591075eabfc58d555a22f619c4575e6#A#compaction#309 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:33,641 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/64d82143674e42978ae0a647741647d9 is 175, key is test_row_0/A:col10/1731752730739/Put/seqid=0 2024-11-16T10:25:33,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742192_1368 (size=31685) 2024-11-16T10:25:33,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T10:25:33,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:33,758 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing a591075eabfc58d555a22f619c4575e6 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-16T10:25:33,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168dc85030c3744e86bc0cfc90efcc8476_a591075eabfc58d555a22f619c4575e6 is 50, key is test_row_0/A:col10/1731752731993/Put/seqid=0 2024-11-16T10:25:33,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742193_1369 (size=12454) 2024-11-16T10:25:33,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:33,771 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168dc85030c3744e86bc0cfc90efcc8476_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168dc85030c3744e86bc0cfc90efcc8476_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:33,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/bbc93a9736b14edcae54f7c9f35e2094, store: [table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:33,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/bbc93a9736b14edcae54f7c9f35e2094 is 175, key is test_row_0/A:col10/1731752731993/Put/seqid=0 2024-11-16T10:25:33,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742194_1370 (size=31255) 2024-11-16T10:25:34,033 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/ea6ebb466e824dc9bd02d63510e76195 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/ea6ebb466e824dc9bd02d63510e76195 2024-11-16T10:25:34,036 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a591075eabfc58d555a22f619c4575e6/B of a591075eabfc58d555a22f619c4575e6 into ea6ebb466e824dc9bd02d63510e76195(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:34,036 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:34,037 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., storeName=a591075eabfc58d555a22f619c4575e6/B, priority=13, startTime=1731752733611; duration=0sec 2024-11-16T10:25:34,037 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:34,037 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a591075eabfc58d555a22f619c4575e6:B 2024-11-16T10:25:34,037 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:25:34,037 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:25:34,037 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): a591075eabfc58d555a22f619c4575e6/C is initiating minor compaction (all files) 2024-11-16T10:25:34,037 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a591075eabfc58d555a22f619c4575e6/C in TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:34,037 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/77a919f2d28e469cbbc11e263916a3e9, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/37f3116f3d54482680e471da70d5f873, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/b4779f951d75483781d97ea2a47b7590] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp, totalSize=36.1 K 2024-11-16T10:25:34,038 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 77a919f2d28e469cbbc11e263916a3e9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731752727908 2024-11-16T10:25:34,038 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 37f3116f3d54482680e471da70d5f873, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731752730566 2024-11-16T10:25:34,038 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting b4779f951d75483781d97ea2a47b7590, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731752730739 2024-11-16T10:25:34,043 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a591075eabfc58d555a22f619c4575e6#C#compaction#311 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:34,044 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/f5cca9a13edc4c7fb6614865a23844bf is 50, key is test_row_0/C:col10/1731752730739/Put/seqid=0 2024-11-16T10:25:34,051 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/64d82143674e42978ae0a647741647d9 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/64d82143674e42978ae0a647741647d9 2024-11-16T10:25:34,054 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a591075eabfc58d555a22f619c4575e6/A of a591075eabfc58d555a22f619c4575e6 into 64d82143674e42978ae0a647741647d9(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:34,055 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:34,055 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., storeName=a591075eabfc58d555a22f619c4575e6/A, priority=13, startTime=1731752733611; duration=0sec 2024-11-16T10:25:34,055 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:34,055 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a591075eabfc58d555a22f619c4575e6:A 2024-11-16T10:25:34,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742195_1371 (size=12731) 2024-11-16T10:25:34,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:34,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:34,179 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/bbc93a9736b14edcae54f7c9f35e2094 2024-11-16T10:25:34,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:34,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752794174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:34,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:34,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752794175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:34,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:34,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:34,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752794175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:34,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:34,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/630b18628c144c999f1ffab39da464e0 is 50, key is test_row_0/B:col10/1731752731993/Put/seqid=0 2024-11-16T10:25:34,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:34,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742196_1372 (size=12301) 2024-11-16T10:25:34,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:34,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752794179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:34,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:34,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:34,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752794180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:34,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:34,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T10:25:34,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:34,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752794286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:34,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:34,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752794286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:34,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:34,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:34,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:34,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752794296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:34,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:34,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752794296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:34,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:34,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:34,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:34,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752794296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:34,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:34,464 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/f5cca9a13edc4c7fb6614865a23844bf as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/f5cca9a13edc4c7fb6614865a23844bf 2024-11-16T10:25:34,467 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a591075eabfc58d555a22f619c4575e6/C of a591075eabfc58d555a22f619c4575e6 into f5cca9a13edc4c7fb6614865a23844bf(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:34,467 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:34,468 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., storeName=a591075eabfc58d555a22f619c4575e6/C, priority=13, startTime=1731752733611; duration=0sec 2024-11-16T10:25:34,468 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:34,468 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a591075eabfc58d555a22f619c4575e6:C 2024-11-16T10:25:34,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:34,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752794496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:34,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:34,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:34,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752794496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:34,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:34,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:34,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752794506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:34,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:34,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:34,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752794506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:34,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:34,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:34,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752794506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:34,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:34,588 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/630b18628c144c999f1ffab39da464e0 2024-11-16T10:25:34,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/129c2c79959f4850a47f10ec4b8864fd is 50, key is test_row_0/C:col10/1731752731993/Put/seqid=0 2024-11-16T10:25:34,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742197_1373 (size=12301) 2024-11-16T10:25:34,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752794806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:34,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:34,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752794806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:34,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:34,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752794816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752794816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752794816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:35,000 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/129c2c79959f4850a47f10ec4b8864fd 2024-11-16T10:25:35,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/bbc93a9736b14edcae54f7c9f35e2094 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/bbc93a9736b14edcae54f7c9f35e2094 2024-11-16T10:25:35,007 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/bbc93a9736b14edcae54f7c9f35e2094, entries=150, sequenceid=275, filesize=30.5 K 2024-11-16T10:25:35,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/630b18628c144c999f1ffab39da464e0 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/630b18628c144c999f1ffab39da464e0 2024-11-16T10:25:35,011 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/630b18628c144c999f1ffab39da464e0, entries=150, sequenceid=275, filesize=12.0 K 2024-11-16T10:25:35,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/129c2c79959f4850a47f10ec4b8864fd as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/129c2c79959f4850a47f10ec4b8864fd 2024-11-16T10:25:35,015 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/129c2c79959f4850a47f10ec4b8864fd, entries=150, sequenceid=275, filesize=12.0 K 2024-11-16T10:25:35,016 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for a591075eabfc58d555a22f619c4575e6 in 1258ms, sequenceid=275, compaction requested=false 2024-11-16T10:25:35,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:35,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:35,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-16T10:25:35,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-16T10:25:35,020 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-16T10:25:35,020 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9390 sec 2024-11-16T10:25:35,022 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.9430 sec 2024-11-16T10:25:35,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:35,320 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:25:35,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a591075eabfc58d555a22f619c4575e6 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-16T10:25:35,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111676e254d0ddc0495cb28cb648caa6a8d3_a591075eabfc58d555a22f619c4575e6 is 50, key is test_row_0/A:col10/1731752735319/Put/seqid=0 2024-11-16T10:25:35,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742198_1374 (size=17534) 2024-11-16T10:25:35,337 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:35,339 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111676e254d0ddc0495cb28cb648caa6a8d3_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111676e254d0ddc0495cb28cb648caa6a8d3_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:35,340 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/d5a18318da094e7b86fc4cdb67c85d36, store: [table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:35,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/d5a18318da094e7b86fc4cdb67c85d36 is 175, key is test_row_0/A:col10/1731752735319/Put/seqid=0 2024-11-16T10:25:35,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742199_1375 (size=48639) 2024-11-16T10:25:35,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752795382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:35,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:35,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:35,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752795391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:35,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:35,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752795391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:35,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:35,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752795393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752795393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:35,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:35,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752795496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:35,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:35,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752795496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:35,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:35,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:35,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:35,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752795506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:35,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:35,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752795516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:35,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:35,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752795516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:35,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:35,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:35,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752795706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:35,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:35,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752795706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:35,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:35,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:35,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752795716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:35,745 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/d5a18318da094e7b86fc4cdb67c85d36 2024-11-16T10:25:35,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:35,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752795736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:35,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:35,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752795737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:35,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:35,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:35,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/1667f121f2ea40d1b355c80566648fc2 is 50, key is test_row_0/B:col10/1731752735319/Put/seqid=0 2024-11-16T10:25:35,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742200_1376 (size=12301) 2024-11-16T10:25:36,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752796017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:36,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:36,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752796017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:36,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:36,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:36,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:36,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:36,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:36,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752796036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:36,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:36,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:36,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:36,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:36,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752796056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:36,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:36,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:36,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752796056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:36,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:36,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:36,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:36,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:36,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:36,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/1667f121f2ea40d1b355c80566648fc2 2024-11-16T10:25:36,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/b0bef8e71cd744f59a0fea21dd92207e is 50, key is test_row_0/C:col10/1731752735319/Put/seqid=0 2024-11-16T10:25:36,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742201_1377 (size=12301) 2024-11-16T10:25:36,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T10:25:36,246 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:25:36,246 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:25:36,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-16T10:25:36,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T10:25:36,248 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:25:36,248 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:25:36,249 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:25:36,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T10:25:36,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T10:25:36,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:36,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:36,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:36,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:36,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:36,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:36,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:36,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1731752796526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:36,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:36,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58562 deadline: 1731752796536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:36,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:36,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:36,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:36,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:36,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1731752796546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:36,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:36,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:36,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:36,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T10:25:36,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:36,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:36,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:36,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:36,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:36,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:36,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T10:25:36,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/b0bef8e71cd744f59a0fea21dd92207e 2024-11-16T10:25:36,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1731752796566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:36,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:36,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:36,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58550 deadline: 1731752796566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:36,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:36,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:36,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:36,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:25:36,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/d5a18318da094e7b86fc4cdb67c85d36 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/d5a18318da094e7b86fc4cdb67c85d36 2024-11-16T10:25:36,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/d5a18318da094e7b86fc4cdb67c85d36, entries=250, sequenceid=294, filesize=47.5 K 2024-11-16T10:25:36,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/1667f121f2ea40d1b355c80566648fc2 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/1667f121f2ea40d1b355c80566648fc2 2024-11-16T10:25:36,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/1667f121f2ea40d1b355c80566648fc2, entries=150, sequenceid=294, filesize=12.0 K 2024-11-16T10:25:36,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/b0bef8e71cd744f59a0fea21dd92207e as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/b0bef8e71cd744f59a0fea21dd92207e 2024-11-16T10:25:36,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/b0bef8e71cd744f59a0fea21dd92207e, entries=150, sequenceid=294, filesize=12.0 K 2024-11-16T10:25:36,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=127.47 KB/130530 for a591075eabfc58d555a22f619c4575e6 in 1264ms, sequenceid=294, compaction requested=true 2024-11-16T10:25:36,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:36,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a591075eabfc58d555a22f619c4575e6:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:25:36,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:36,584 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:25:36,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a591075eabfc58d555a22f619c4575e6:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:25:36,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:36,584 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:25:36,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a591075eabfc58d555a22f619c4575e6:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:25:36,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:36,585 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:25:36,585 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111579 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:25:36,585 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): a591075eabfc58d555a22f619c4575e6/B is initiating minor compaction (all files) 2024-11-16T10:25:36,585 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): a591075eabfc58d555a22f619c4575e6/A is initiating minor compaction (all files) 2024-11-16T10:25:36,585 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a591075eabfc58d555a22f619c4575e6/A in TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:36,585 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a591075eabfc58d555a22f619c4575e6/B in TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:36,585 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/64d82143674e42978ae0a647741647d9, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/bbc93a9736b14edcae54f7c9f35e2094, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/d5a18318da094e7b86fc4cdb67c85d36] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp, totalSize=109.0 K 2024-11-16T10:25:36,585 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/ea6ebb466e824dc9bd02d63510e76195, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/630b18628c144c999f1ffab39da464e0, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/1667f121f2ea40d1b355c80566648fc2] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp, totalSize=36.5 K 2024-11-16T10:25:36,585 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:36,585 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. files: [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/64d82143674e42978ae0a647741647d9, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/bbc93a9736b14edcae54f7c9f35e2094, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/d5a18318da094e7b86fc4cdb67c85d36] 2024-11-16T10:25:36,585 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64d82143674e42978ae0a647741647d9, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731752730739 2024-11-16T10:25:36,585 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting ea6ebb466e824dc9bd02d63510e76195, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731752730739 2024-11-16T10:25:36,586 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 630b18628c144c999f1ffab39da464e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731752731993 2024-11-16T10:25:36,586 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbc93a9736b14edcae54f7c9f35e2094, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731752731993 2024-11-16T10:25:36,586 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5a18318da094e7b86fc4cdb67c85d36, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731752734172 2024-11-16T10:25:36,586 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 1667f121f2ea40d1b355c80566648fc2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731752734172 2024-11-16T10:25:36,590 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:36,591 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a591075eabfc58d555a22f619c4575e6#B#compaction#317 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:36,591 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/1c40f399f91747e6b619797fafdac584 is 50, key is test_row_0/B:col10/1731752735319/Put/seqid=0 2024-11-16T10:25:36,592 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116a36f014664674c6ba02b52d436ff45c8_a591075eabfc58d555a22f619c4575e6 store=[table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:36,594 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116a36f014664674c6ba02b52d436ff45c8_a591075eabfc58d555a22f619c4575e6, store=[table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:36,594 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a36f014664674c6ba02b52d436ff45c8_a591075eabfc58d555a22f619c4575e6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:36,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742202_1378 (size=12983) 2024-11-16T10:25:36,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742203_1379 (size=4469) 2024-11-16T10:25:36,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T10:25:36,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:36,706 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing a591075eabfc58d555a22f619c4575e6 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-16T10:25:36,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e1723c4e9af54b4083b961c50d8b81da_a591075eabfc58d555a22f619c4575e6 is 50, key is test_row_0/A:col10/1731752735390/Put/seqid=0 2024-11-16T10:25:36,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742204_1380 (size=12454) 2024-11-16T10:25:36,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T10:25:36,999 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a591075eabfc58d555a22f619c4575e6#A#compaction#318 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:36,999 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/eea8012c99ba423db74a882d959ab8c0 is 175, key is test_row_0/A:col10/1731752735319/Put/seqid=0 2024-11-16T10:25:37,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742205_1381 (size=31937) 2024-11-16T10:25:37,005 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/1c40f399f91747e6b619797fafdac584 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/1c40f399f91747e6b619797fafdac584 2024-11-16T10:25:37,009 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a591075eabfc58d555a22f619c4575e6/B of a591075eabfc58d555a22f619c4575e6 into 1c40f399f91747e6b619797fafdac584(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:37,009 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:37,009 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., storeName=a591075eabfc58d555a22f619c4575e6/B, priority=13, startTime=1731752736584; duration=0sec 2024-11-16T10:25:37,009 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:37,009 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a591075eabfc58d555a22f619c4575e6:B 2024-11-16T10:25:37,009 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:25:37,010 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:25:37,010 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): a591075eabfc58d555a22f619c4575e6/C is initiating minor compaction (all files) 2024-11-16T10:25:37,010 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a591075eabfc58d555a22f619c4575e6/C in TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:37,011 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/f5cca9a13edc4c7fb6614865a23844bf, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/129c2c79959f4850a47f10ec4b8864fd, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/b0bef8e71cd744f59a0fea21dd92207e] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp, totalSize=36.5 K 2024-11-16T10:25:37,015 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting f5cca9a13edc4c7fb6614865a23844bf, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731752730739 2024-11-16T10:25:37,016 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 129c2c79959f4850a47f10ec4b8864fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731752731993 2024-11-16T10:25:37,016 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting b0bef8e71cd744f59a0fea21dd92207e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731752734172 2024-11-16T10:25:37,021 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a591075eabfc58d555a22f619c4575e6#C#compaction#320 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:37,022 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/380986ff823f49dfbea4ec606973ea4d is 50, key is test_row_0/C:col10/1731752735319/Put/seqid=0 2024-11-16T10:25:37,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742206_1382 (size=12983) 2024-11-16T10:25:37,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:37,123 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e1723c4e9af54b4083b961c50d8b81da_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e1723c4e9af54b4083b961c50d8b81da_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:37,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/7225abdc755041e38105b267f3d2b59a, store: [table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:37,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/7225abdc755041e38105b267f3d2b59a is 175, key is test_row_0/A:col10/1731752735390/Put/seqid=0 2024-11-16T10:25:37,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742207_1383 (size=31255) 2024-11-16T10:25:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T10:25:37,409 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/eea8012c99ba423db74a882d959ab8c0 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/eea8012c99ba423db74a882d959ab8c0 2024-11-16T10:25:37,413 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a591075eabfc58d555a22f619c4575e6/A of a591075eabfc58d555a22f619c4575e6 into eea8012c99ba423db74a882d959ab8c0(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:37,413 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:37,413 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., storeName=a591075eabfc58d555a22f619c4575e6/A, priority=13, startTime=1731752736584; duration=0sec 2024-11-16T10:25:37,413 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:37,413 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a591075eabfc58d555a22f619c4575e6:A 2024-11-16T10:25:37,433 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/380986ff823f49dfbea4ec606973ea4d as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/380986ff823f49dfbea4ec606973ea4d 2024-11-16T10:25:37,436 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a591075eabfc58d555a22f619c4575e6/C of a591075eabfc58d555a22f619c4575e6 into 380986ff823f49dfbea4ec606973ea4d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:37,436 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:37,437 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., storeName=a591075eabfc58d555a22f619c4575e6/C, priority=13, startTime=1731752736584; duration=0sec 2024-11-16T10:25:37,437 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:37,437 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a591075eabfc58d555a22f619c4575e6:C 2024-11-16T10:25:37,460 INFO [Thread-1388 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1388. 2024-11-16T10:25:37,460 DEBUG [Thread-1388 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:25:37,460 DEBUG [Thread-1388 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:37,460 DEBUG [Thread-1388 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:37,460 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:25:37,462 INFO [Thread-1392 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1392. 2024-11-16T10:25:37,463 DEBUG [Thread-1392 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:25:37,463 DEBUG [Thread-1392 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:37,463 DEBUG [Thread-1392 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:37,463 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:25:37,463 INFO [Thread-1394 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1394. 2024-11-16T10:25:37,463 DEBUG [Thread-1394 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:25:37,463 DEBUG [Thread-1394 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:37,463 DEBUG [Thread-1394 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:37,463 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:25:37,465 INFO [Thread-1396 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1396. 2024-11-16T10:25:37,465 DEBUG [Thread-1396 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:25:37,465 DEBUG [Thread-1396 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:37,465 DEBUG [Thread-1396 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:37,465 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:25:37,467 INFO [Thread-1390 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1390. 2024-11-16T10:25:37,467 DEBUG [Thread-1390 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:25:37,468 DEBUG [Thread-1390 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:37,468 DEBUG [Thread-1390 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:37,468 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:25:37,527 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/7225abdc755041e38105b267f3d2b59a 2024-11-16T10:25:37,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/0dd453b6b01344d4ad468d5ef5ca7032 is 50, key is test_row_0/B:col10/1731752735390/Put/seqid=0 2024-11-16T10:25:37,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742208_1384 (size=12301) 2024-11-16T10:25:37,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:37,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. as already flushing 2024-11-16T10:25:37,549 INFO [Thread-1377 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1377. 2024-11-16T10:25:37,549 DEBUG [Thread-1377 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:25:37,549 DEBUG [Thread-1377 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:37,549 DEBUG [Thread-1377 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:37,549 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:25:37,557 INFO [Thread-1383 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1383. 2024-11-16T10:25:37,557 INFO [Thread-1379 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1379. 2024-11-16T10:25:37,557 DEBUG [Thread-1383 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:25:37,557 DEBUG [Thread-1379 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:25:37,557 DEBUG [Thread-1379 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:37,557 DEBUG [Thread-1383 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:37,557 DEBUG [Thread-1383 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:37,557 DEBUG [Thread-1379 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:37,557 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:25:37,557 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:25:37,589 INFO [Thread-1381 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1381. 2024-11-16T10:25:37,589 INFO [Thread-1385 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1385. 2024-11-16T10:25:37,589 DEBUG [Thread-1381 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:25:37,589 DEBUG [Thread-1385 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:25:37,589 DEBUG [Thread-1381 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:37,589 DEBUG [Thread-1385 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:37,589 DEBUG [Thread-1385 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:37,589 DEBUG [Thread-1381 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:37,589 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:25:37,589 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:25:37,945 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/0dd453b6b01344d4ad468d5ef5ca7032 2024-11-16T10:25:37,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/6355dab229214ed79ea413d98d0ead10 is 50, key is test_row_0/C:col10/1731752735390/Put/seqid=0 2024-11-16T10:25:37,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742209_1385 (size=12301) 2024-11-16T10:25:38,363 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/6355dab229214ed79ea413d98d0ead10 2024-11-16T10:25:38,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/7225abdc755041e38105b267f3d2b59a as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/7225abdc755041e38105b267f3d2b59a 2024-11-16T10:25:38,380 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/7225abdc755041e38105b267f3d2b59a, entries=150, sequenceid=316, filesize=30.5 K 2024-11-16T10:25:38,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/0dd453b6b01344d4ad468d5ef5ca7032 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/0dd453b6b01344d4ad468d5ef5ca7032 2024-11-16T10:25:38,384 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/0dd453b6b01344d4ad468d5ef5ca7032, entries=150, sequenceid=316, filesize=12.0 K 2024-11-16T10:25:38,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/6355dab229214ed79ea413d98d0ead10 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/6355dab229214ed79ea413d98d0ead10 2024-11-16T10:25:38,388 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/6355dab229214ed79ea413d98d0ead10, entries=150, sequenceid=316, filesize=12.0 K 2024-11-16T10:25:38,389 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=33.54 KB/34350 for a591075eabfc58d555a22f619c4575e6 in 1683ms, sequenceid=316, compaction requested=false 2024-11-16T10:25:38,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for a591075eabfc58d555a22f619c4575e6: 2024-11-16T10:25:38,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:38,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-16T10:25:38,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-16T10:25:38,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-16T10:25:38,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1410 sec 2024-11-16T10:25:38,392 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.1460 sec 2024-11-16T10:25:38,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T10:25:38,396 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:25:38,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T10:25:38,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-16T10:25:38,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-16T10:25:38,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-16T10:25:38,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-16T10:25:38,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-16T10:25:38,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T10:25:38,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T10:25:38,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1224 2024-11-16T10:25:38,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3647 rows 2024-11-16T10:25:38,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1217 2024-11-16T10:25:38,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3642 rows 2024-11-16T10:25:38,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1221 2024-11-16T10:25:38,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3640 rows 2024-11-16T10:25:38,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1206 2024-11-16T10:25:38,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3608 rows 2024-11-16T10:25:38,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1215 2024-11-16T10:25:38,396 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3644 rows 2024-11-16T10:25:38,396 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T10:25:38,396 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T10:25:38,396 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:38,396 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:38,396 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:25:38,397 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T10:25:38,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T10:25:38,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T10:25:38,400 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752738400"}]},"ts":"1731752738400"} 2024-11-16T10:25:38,401 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T10:25:38,401 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T10:25:38,401 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T10:25:38,402 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a591075eabfc58d555a22f619c4575e6, UNASSIGN}] 2024-11-16T10:25:38,403 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a591075eabfc58d555a22f619c4575e6, UNASSIGN 2024-11-16T10:25:38,404 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=a591075eabfc58d555a22f619c4575e6, regionState=CLOSING, regionLocation=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:38,405 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a591075eabfc58d555a22f619c4575e6, UNASSIGN because future has completed 2024-11-16T10:25:38,405 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T10:25:38,405 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506}] 2024-11-16T10:25:38,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T10:25:38,557 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(122): Close a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:38,557 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T10:25:38,557 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1722): Closing a591075eabfc58d555a22f619c4575e6, disabling compactions & flushes 2024-11-16T10:25:38,557 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:38,557 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:38,557 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. after waiting 0 ms 2024-11-16T10:25:38,557 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:38,557 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(2902): Flushing a591075eabfc58d555a22f619c4575e6 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-16T10:25:38,562 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fde786c7d3474f10a467eefa692f15d8_a591075eabfc58d555a22f619c4575e6 is 50, key is test_row_0/A:col10/1731752737547/Put/seqid=0 2024-11-16T10:25:38,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742210_1386 (size=9914) 2024-11-16T10:25:38,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T10:25:38,967 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:38,975 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fde786c7d3474f10a467eefa692f15d8_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fde786c7d3474f10a467eefa692f15d8_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:38,976 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/9f70c9afcc1a49fdb1c539dd09c3f1f2, store: [table=TestAcidGuarantees family=A region=a591075eabfc58d555a22f619c4575e6] 2024-11-16T10:25:38,977 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/9f70c9afcc1a49fdb1c539dd09c3f1f2 is 175, key is test_row_0/A:col10/1731752737547/Put/seqid=0 2024-11-16T10:25:38,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742211_1387 (size=22561) 2024-11-16T10:25:39,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T10:25:39,382 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=327, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/9f70c9afcc1a49fdb1c539dd09c3f1f2 2024-11-16T10:25:39,396 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/f904ae13b1b14f7195d121eb9faaaafb is 50, key is test_row_0/B:col10/1731752737547/Put/seqid=0 2024-11-16T10:25:39,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742212_1388 (size=9857) 2024-11-16T10:25:39,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T10:25:39,803 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/f904ae13b1b14f7195d121eb9faaaafb 2024-11-16T10:25:39,817 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/ea77fc06ef0945f99798f4aad36cc2a5 is 50, key is test_row_0/C:col10/1731752737547/Put/seqid=0 2024-11-16T10:25:39,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742213_1389 (size=9857) 2024-11-16T10:25:40,222 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/ea77fc06ef0945f99798f4aad36cc2a5 2024-11-16T10:25:40,233 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/A/9f70c9afcc1a49fdb1c539dd09c3f1f2 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/9f70c9afcc1a49fdb1c539dd09c3f1f2 2024-11-16T10:25:40,238 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/9f70c9afcc1a49fdb1c539dd09c3f1f2, entries=100, sequenceid=327, filesize=22.0 K 2024-11-16T10:25:40,239 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/B/f904ae13b1b14f7195d121eb9faaaafb as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/f904ae13b1b14f7195d121eb9faaaafb 2024-11-16T10:25:40,243 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/f904ae13b1b14f7195d121eb9faaaafb, entries=100, sequenceid=327, filesize=9.6 K 2024-11-16T10:25:40,244 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/.tmp/C/ea77fc06ef0945f99798f4aad36cc2a5 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/ea77fc06ef0945f99798f4aad36cc2a5 2024-11-16T10:25:40,248 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/ea77fc06ef0945f99798f4aad36cc2a5, entries=100, sequenceid=327, filesize=9.6 K 2024-11-16T10:25:40,249 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for a591075eabfc58d555a22f619c4575e6 in 1692ms, sequenceid=327, compaction requested=true 2024-11-16T10:25:40,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/92f608ec54c74032b24878b046f302b9, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/46fbe02d9beb4e9989448dcd40213f23, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/32baae1b021d43c68a13aa5cabb67e83, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/0b5c3d8dd19e43cab2eeeacd4bfc463e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/36c1475b40334d879c2041ece5c4225f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/4a7764e881914c678daa292d99d8b3da, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/9e0542df85584031ba2e83115c67f69b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/48f685f2bf1d4ea7b8c36fbcad5613a0, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/e9ebc91377694adaafe686ef70a1fef0, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/a7a303b46b4e49c4b427d5373b0fffee, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/95f58abf786147908bb63f35c444034c, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/1ee34b8bb37c43eb8e8be30ab5f88293, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/65ab26bd7f0242e888754d62b2daea2d, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/1855c6c68c4b407f89b9edf60ffc0d6c, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/54e0cac28c144af2861f348aa2ebef21, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/33662939d3374bf49cff007974669f4d, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/3e61c85967e24b819027d66b4e5caca6, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/64d82143674e42978ae0a647741647d9, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/bbc93a9736b14edcae54f7c9f35e2094, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/d5a18318da094e7b86fc4cdb67c85d36] to archive 2024-11-16T10:25:40,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:25:40,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/92f608ec54c74032b24878b046f302b9 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/92f608ec54c74032b24878b046f302b9 2024-11-16T10:25:40,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/46fbe02d9beb4e9989448dcd40213f23 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/46fbe02d9beb4e9989448dcd40213f23 2024-11-16T10:25:40,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/32baae1b021d43c68a13aa5cabb67e83 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/32baae1b021d43c68a13aa5cabb67e83 2024-11-16T10:25:40,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/0b5c3d8dd19e43cab2eeeacd4bfc463e to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/0b5c3d8dd19e43cab2eeeacd4bfc463e 2024-11-16T10:25:40,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/36c1475b40334d879c2041ece5c4225f to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/36c1475b40334d879c2041ece5c4225f 2024-11-16T10:25:40,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/4a7764e881914c678daa292d99d8b3da to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/4a7764e881914c678daa292d99d8b3da 2024-11-16T10:25:40,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/9e0542df85584031ba2e83115c67f69b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/9e0542df85584031ba2e83115c67f69b 2024-11-16T10:25:40,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/48f685f2bf1d4ea7b8c36fbcad5613a0 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/48f685f2bf1d4ea7b8c36fbcad5613a0 2024-11-16T10:25:40,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/e9ebc91377694adaafe686ef70a1fef0 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/e9ebc91377694adaafe686ef70a1fef0 2024-11-16T10:25:40,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/a7a303b46b4e49c4b427d5373b0fffee to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/a7a303b46b4e49c4b427d5373b0fffee 2024-11-16T10:25:40,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/95f58abf786147908bb63f35c444034c to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/95f58abf786147908bb63f35c444034c 2024-11-16T10:25:40,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/1ee34b8bb37c43eb8e8be30ab5f88293 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/1ee34b8bb37c43eb8e8be30ab5f88293 2024-11-16T10:25:40,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/65ab26bd7f0242e888754d62b2daea2d to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/65ab26bd7f0242e888754d62b2daea2d 2024-11-16T10:25:40,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/1855c6c68c4b407f89b9edf60ffc0d6c to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/1855c6c68c4b407f89b9edf60ffc0d6c 2024-11-16T10:25:40,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/54e0cac28c144af2861f348aa2ebef21 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/54e0cac28c144af2861f348aa2ebef21 2024-11-16T10:25:40,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/33662939d3374bf49cff007974669f4d to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/33662939d3374bf49cff007974669f4d 2024-11-16T10:25:40,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/3e61c85967e24b819027d66b4e5caca6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/3e61c85967e24b819027d66b4e5caca6 2024-11-16T10:25:40,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/64d82143674e42978ae0a647741647d9 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/64d82143674e42978ae0a647741647d9 2024-11-16T10:25:40,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/bbc93a9736b14edcae54f7c9f35e2094 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/bbc93a9736b14edcae54f7c9f35e2094 2024-11-16T10:25:40,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/d5a18318da094e7b86fc4cdb67c85d36 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/d5a18318da094e7b86fc4cdb67c85d36 2024-11-16T10:25:40,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/16c4dcac7b6141ceb56b27d9d38805a0, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/0be6c4c45b234533a55c9631987fac4e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/80dd666d473b41e5a82545942b9ddd4f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/0f36fed28754404390572fa491d35756, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/57c5e1e228354f79a1fa4e8d4a0d7efb, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/b572394d9b614a11a0009785bff46183, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/b974048b9b90443bb779258b85891b92, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/4427a74a93924a49b9ed68e8c300f54c, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/b204e83bea7942fda9dcc86d76dcb92a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/23a829e35c92458d906207cd582ea200, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/eb5914493fad45caae5a38378734e9dc, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/248710cd6ebd41a996f090d44a3e6f6f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/afb0865d11a44995b0869d8513f90924, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/4cf9dd56da90408db7720ddf7236853a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/a7264bc598474e42b4b82f1471e5eb4c, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/efda052e7c5c4256b651908ea1fc74c4, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/ea6ebb466e824dc9bd02d63510e76195, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/79763c1ed7af4f4aa1a6353c5ae9d00f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/630b18628c144c999f1ffab39da464e0, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/1667f121f2ea40d1b355c80566648fc2] to archive 2024-11-16T10:25:40,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:25:40,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/16c4dcac7b6141ceb56b27d9d38805a0 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/16c4dcac7b6141ceb56b27d9d38805a0 2024-11-16T10:25:40,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/0be6c4c45b234533a55c9631987fac4e to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/0be6c4c45b234533a55c9631987fac4e 2024-11-16T10:25:40,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/80dd666d473b41e5a82545942b9ddd4f to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/80dd666d473b41e5a82545942b9ddd4f 2024-11-16T10:25:40,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/0f36fed28754404390572fa491d35756 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/0f36fed28754404390572fa491d35756 2024-11-16T10:25:40,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/57c5e1e228354f79a1fa4e8d4a0d7efb to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/57c5e1e228354f79a1fa4e8d4a0d7efb 2024-11-16T10:25:40,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/b572394d9b614a11a0009785bff46183 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/b572394d9b614a11a0009785bff46183 2024-11-16T10:25:40,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/b974048b9b90443bb779258b85891b92 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/b974048b9b90443bb779258b85891b92 2024-11-16T10:25:40,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/4427a74a93924a49b9ed68e8c300f54c to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/4427a74a93924a49b9ed68e8c300f54c 2024-11-16T10:25:40,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/b204e83bea7942fda9dcc86d76dcb92a to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/b204e83bea7942fda9dcc86d76dcb92a 2024-11-16T10:25:40,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/23a829e35c92458d906207cd582ea200 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/23a829e35c92458d906207cd582ea200 2024-11-16T10:25:40,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/eb5914493fad45caae5a38378734e9dc to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/eb5914493fad45caae5a38378734e9dc 2024-11-16T10:25:40,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/248710cd6ebd41a996f090d44a3e6f6f to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/248710cd6ebd41a996f090d44a3e6f6f 2024-11-16T10:25:40,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/afb0865d11a44995b0869d8513f90924 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/afb0865d11a44995b0869d8513f90924 2024-11-16T10:25:40,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/4cf9dd56da90408db7720ddf7236853a to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/4cf9dd56da90408db7720ddf7236853a 2024-11-16T10:25:40,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/a7264bc598474e42b4b82f1471e5eb4c to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/a7264bc598474e42b4b82f1471e5eb4c 2024-11-16T10:25:40,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/efda052e7c5c4256b651908ea1fc74c4 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/efda052e7c5c4256b651908ea1fc74c4 2024-11-16T10:25:40,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/ea6ebb466e824dc9bd02d63510e76195 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/ea6ebb466e824dc9bd02d63510e76195 2024-11-16T10:25:40,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/79763c1ed7af4f4aa1a6353c5ae9d00f to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/79763c1ed7af4f4aa1a6353c5ae9d00f 2024-11-16T10:25:40,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/630b18628c144c999f1ffab39da464e0 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/630b18628c144c999f1ffab39da464e0 2024-11-16T10:25:40,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/1667f121f2ea40d1b355c80566648fc2 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/1667f121f2ea40d1b355c80566648fc2 2024-11-16T10:25:40,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/62d408e98454446ebb02ef5388296b92, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/2207e4f531a84edda995e630f4f44fe4, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/d3b90a0ca5b84011abee5ade25fb6ae3, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/80542add2d0f45deb845a1c6194350f4, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/48b97f641bea4e9cbe133e8c31604dd8, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/0ddc825308654760a3cfd9fdee9b1d77, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/6f226f2739834c2bb1b931635d206c73, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/5dbd2a08dcbd44d48b68bace0a243841, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/4f3a17450f454857b94d1e1a14127d12, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/288c39efec77423b86aa0e5def9d44e6, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/99114140026e4067a1eeb8203e16bf52, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/039dfcb6b5174bef933adec064e8607b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/6054268fb8cd4525a00858de8b18630b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/77a919f2d28e469cbbc11e263916a3e9, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/9d98439aeb904d259f45ac1e269785cc, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/37f3116f3d54482680e471da70d5f873, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/f5cca9a13edc4c7fb6614865a23844bf, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/b4779f951d75483781d97ea2a47b7590, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/129c2c79959f4850a47f10ec4b8864fd, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/b0bef8e71cd744f59a0fea21dd92207e] to archive 2024-11-16T10:25:40,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:25:40,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/62d408e98454446ebb02ef5388296b92 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/62d408e98454446ebb02ef5388296b92 2024-11-16T10:25:40,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/2207e4f531a84edda995e630f4f44fe4 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/2207e4f531a84edda995e630f4f44fe4 2024-11-16T10:25:40,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/d3b90a0ca5b84011abee5ade25fb6ae3 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/d3b90a0ca5b84011abee5ade25fb6ae3 2024-11-16T10:25:40,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/80542add2d0f45deb845a1c6194350f4 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/80542add2d0f45deb845a1c6194350f4 2024-11-16T10:25:40,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/48b97f641bea4e9cbe133e8c31604dd8 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/48b97f641bea4e9cbe133e8c31604dd8 2024-11-16T10:25:40,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/0ddc825308654760a3cfd9fdee9b1d77 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/0ddc825308654760a3cfd9fdee9b1d77 2024-11-16T10:25:40,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/6f226f2739834c2bb1b931635d206c73 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/6f226f2739834c2bb1b931635d206c73 2024-11-16T10:25:40,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/5dbd2a08dcbd44d48b68bace0a243841 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/5dbd2a08dcbd44d48b68bace0a243841 2024-11-16T10:25:40,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/4f3a17450f454857b94d1e1a14127d12 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/4f3a17450f454857b94d1e1a14127d12 2024-11-16T10:25:40,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/288c39efec77423b86aa0e5def9d44e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/288c39efec77423b86aa0e5def9d44e6 2024-11-16T10:25:40,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/99114140026e4067a1eeb8203e16bf52 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/99114140026e4067a1eeb8203e16bf52 2024-11-16T10:25:40,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/039dfcb6b5174bef933adec064e8607b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/039dfcb6b5174bef933adec064e8607b 2024-11-16T10:25:40,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/6054268fb8cd4525a00858de8b18630b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/6054268fb8cd4525a00858de8b18630b 2024-11-16T10:25:40,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/77a919f2d28e469cbbc11e263916a3e9 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/77a919f2d28e469cbbc11e263916a3e9 2024-11-16T10:25:40,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/9d98439aeb904d259f45ac1e269785cc to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/9d98439aeb904d259f45ac1e269785cc 2024-11-16T10:25:40,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/37f3116f3d54482680e471da70d5f873 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/37f3116f3d54482680e471da70d5f873 2024-11-16T10:25:40,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/f5cca9a13edc4c7fb6614865a23844bf to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/f5cca9a13edc4c7fb6614865a23844bf 2024-11-16T10:25:40,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/b4779f951d75483781d97ea2a47b7590 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/b4779f951d75483781d97ea2a47b7590 2024-11-16T10:25:40,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/129c2c79959f4850a47f10ec4b8864fd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/129c2c79959f4850a47f10ec4b8864fd 2024-11-16T10:25:40,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/b0bef8e71cd744f59a0fea21dd92207e to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/b0bef8e71cd744f59a0fea21dd92207e 2024-11-16T10:25:40,310 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/recovered.edits/330.seqid, newMaxSeqId=330, maxSeqId=4 2024-11-16T10:25:40,310 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. 2024-11-16T10:25:40,310 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1676): Region close journal for a591075eabfc58d555a22f619c4575e6: Waiting for close lock at 1731752738557Running coprocessor pre-close hooks at 1731752738557Disabling compacts and flushes for region at 1731752738557Disabling writes for close at 1731752738557Obtaining lock to block concurrent updates at 1731752738557Preparing flush snapshotting stores in a591075eabfc58d555a22f619c4575e6 at 1731752738557Finished memstore snapshotting TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731752738557Flushing stores of TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6. at 1731752738558 (+1 ms)Flushing a591075eabfc58d555a22f619c4575e6/A: creating writer at 1731752738558Flushing mob file a591075eabfc58d555a22f619c4575e6/A: appending metadata at 1731752738562 (+4 ms)Flushing mob file a591075eabfc58d555a22f619c4575e6/A: closing flushed file at 1731752738562Flushing a591075eabfc58d555a22f619c4575e6/A: appending metadata at 1731752738976 (+414 ms)Flushing a591075eabfc58d555a22f619c4575e6/A: closing flushed file at 1731752738977 (+1 ms)Flushing a591075eabfc58d555a22f619c4575e6/B: creating writer at 1731752739390 (+413 ms)Flushing a591075eabfc58d555a22f619c4575e6/B: appending metadata at 1731752739395 (+5 ms)Flushing a591075eabfc58d555a22f619c4575e6/B: closing flushed file at 1731752739395Flushing a591075eabfc58d555a22f619c4575e6/C: creating writer at 1731752739813 (+418 ms)Flushing a591075eabfc58d555a22f619c4575e6/C: appending metadata at 1731752739816 (+3 ms)Flushing a591075eabfc58d555a22f619c4575e6/C: closing flushed file at 1731752739816Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b35bdef: reopening flushed file at 1731752740231 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8ae72db: reopening flushed file at 1731752740238 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65d7dcc0: reopening flushed file at 1731752740243 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for a591075eabfc58d555a22f619c4575e6 in 1692ms, sequenceid=327, compaction requested=true at 1731752740249 (+6 ms)Writing region close event to WAL at 1731752740307 (+58 ms)Running coprocessor post-close hooks at 1731752740310 (+3 ms)Closed at 1731752740310 2024-11-16T10:25:40,312 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(157): Closed a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:40,313 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=a591075eabfc58d555a22f619c4575e6, regionState=CLOSED 2024-11-16T10:25:40,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 because future has completed 2024-11-16T10:25:40,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-16T10:25:40,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseRegionProcedure a591075eabfc58d555a22f619c4575e6, server=b16ff38458c1,35873,1731752627506 in 1.9100 sec 2024-11-16T10:25:40,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-11-16T10:25:40,319 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a591075eabfc58d555a22f619c4575e6, UNASSIGN in 1.9150 sec 2024-11-16T10:25:40,321 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-16T10:25:40,321 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9180 sec 2024-11-16T10:25:40,322 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752740322"}]},"ts":"1731752740322"} 2024-11-16T10:25:40,323 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T10:25:40,323 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T10:25:40,325 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9270 sec 2024-11-16T10:25:40,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T10:25:40,547 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:25:40,548 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T10:25:40,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:25:40,550 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:25:40,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T10:25:40,551 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=111, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:25:40,556 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:40,560 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A, FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B, FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C, FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/recovered.edits] 2024-11-16T10:25:40,563 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/7225abdc755041e38105b267f3d2b59a to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/7225abdc755041e38105b267f3d2b59a 2024-11-16T10:25:40,564 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/9f70c9afcc1a49fdb1c539dd09c3f1f2 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/9f70c9afcc1a49fdb1c539dd09c3f1f2 2024-11-16T10:25:40,565 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/eea8012c99ba423db74a882d959ab8c0 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/A/eea8012c99ba423db74a882d959ab8c0 2024-11-16T10:25:40,567 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/0dd453b6b01344d4ad468d5ef5ca7032 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/0dd453b6b01344d4ad468d5ef5ca7032 2024-11-16T10:25:40,568 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/1c40f399f91747e6b619797fafdac584 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/1c40f399f91747e6b619797fafdac584 2024-11-16T10:25:40,569 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/f904ae13b1b14f7195d121eb9faaaafb to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/B/f904ae13b1b14f7195d121eb9faaaafb 2024-11-16T10:25:40,572 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/380986ff823f49dfbea4ec606973ea4d to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/380986ff823f49dfbea4ec606973ea4d 2024-11-16T10:25:40,573 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/6355dab229214ed79ea413d98d0ead10 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/6355dab229214ed79ea413d98d0ead10 2024-11-16T10:25:40,574 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/ea77fc06ef0945f99798f4aad36cc2a5 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/C/ea77fc06ef0945f99798f4aad36cc2a5 2024-11-16T10:25:40,577 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/recovered.edits/330.seqid to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6/recovered.edits/330.seqid 2024-11-16T10:25:40,577 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:40,577 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T10:25:40,578 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T10:25:40,579 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-16T10:25:40,582 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111619422f5a40fe4285aab260be53ac3b58_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111619422f5a40fe4285aab260be53ac3b58_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:40,583 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111620e2b783539f437f9b70a272a074500f_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111620e2b783539f437f9b70a272a074500f_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:40,584 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162def8e306a1d4bb6baefef95fd31708d_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162def8e306a1d4bb6baefef95fd31708d_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:40,586 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111643e191a40d8c4333ac0f1f75b18b0060_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111643e191a40d8c4333ac0f1f75b18b0060_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:40,587 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165bd2a810c6b948db881de81b13cfca73_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165bd2a810c6b948db881de81b13cfca73_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:40,588 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167212da567e00473cad10208549a2259f_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167212da567e00473cad10208549a2259f_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:40,590 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111676e254d0ddc0495cb28cb648caa6a8d3_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111676e254d0ddc0495cb28cb648caa6a8d3_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:40,591 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167aa8de3de2304be489571dc663ae3fc0_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167aa8de3de2304be489571dc663ae3fc0_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:40,592 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168dc85030c3744e86bc0cfc90efcc8476_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168dc85030c3744e86bc0cfc90efcc8476_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:40,593 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116abde551077b840cb9dc9a39da8504d8c_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116abde551077b840cb9dc9a39da8504d8c_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:40,595 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c138a3d2d79049d6aa23a70e5fd3bfab_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c138a3d2d79049d6aa23a70e5fd3bfab_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:40,596 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d11f9b40708c4d2ab18d598a85f1793e_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d11f9b40708c4d2ab18d598a85f1793e_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:40,597 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116de8bf68b5b714ad29fe85be90d22af68_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116de8bf68b5b714ad29fe85be90d22af68_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:40,599 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e1723c4e9af54b4083b961c50d8b81da_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e1723c4e9af54b4083b961c50d8b81da_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:40,600 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116eb8b0d7faa1941d1b75ba66064fc708d_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116eb8b0d7faa1941d1b75ba66064fc708d_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:40,601 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fcbde42a565748bfbd13e3c95d22122e_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fcbde42a565748bfbd13e3c95d22122e_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:40,603 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fde786c7d3474f10a467eefa692f15d8_a591075eabfc58d555a22f619c4575e6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fde786c7d3474f10a467eefa692f15d8_a591075eabfc58d555a22f619c4575e6 2024-11-16T10:25:40,603 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T10:25:40,606 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=111, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:25:40,608 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T10:25:40,610 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T10:25:40,612 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=111, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:25:40,612 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T10:25:40,612 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731752740612"}]},"ts":"9223372036854775807"} 2024-11-16T10:25:40,614 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T10:25:40,614 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a591075eabfc58d555a22f619c4575e6, NAME => 'TestAcidGuarantees,,1731752714293.a591075eabfc58d555a22f619c4575e6.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T10:25:40,614 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T10:25:40,615 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731752740614"}]},"ts":"9223372036854775807"} 2024-11-16T10:25:40,616 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T10:25:40,617 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=111, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:25:40,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 69 msec 2024-11-16T10:25:40,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T10:25:40,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T10:25:40,658 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:25:40,674 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242 (was 244), OpenFileDescriptor=461 (was 469), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=497 (was 481) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4054 (was 4199) 2024-11-16T10:25:40,684 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=242, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=497, ProcessCount=11, AvailableMemoryMB=4053 2024-11-16T10:25:40,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T10:25:40,686 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T10:25:40,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T10:25:40,688 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T10:25:40,688 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:40,688 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 112 2024-11-16T10:25:40,689 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T10:25:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-16T10:25:40,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742214_1390 (size=959) 2024-11-16T10:25:40,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-16T10:25:41,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-16T10:25:41,099 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7 2024-11-16T10:25:41,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742215_1391 (size=53) 2024-11-16T10:25:41,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-16T10:25:41,510 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:25:41,510 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b6c1e23082935a9ab78c6a0d7f447d65, disabling compactions & flushes 2024-11-16T10:25:41,510 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:41,510 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:41,510 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. after waiting 0 ms 2024-11-16T10:25:41,510 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:41,510 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:41,510 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b6c1e23082935a9ab78c6a0d7f447d65: Waiting for close lock at 1731752741510Disabling compacts and flushes for region at 1731752741510Disabling writes for close at 1731752741510Writing region close event to WAL at 1731752741510Closed at 1731752741510 2024-11-16T10:25:41,548 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T10:25:41,548 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731752741548"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731752741548"}]},"ts":"1731752741548"} 2024-11-16T10:25:41,550 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T10:25:41,551 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T10:25:41,551 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752741551"}]},"ts":"1731752741551"} 2024-11-16T10:25:41,553 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T10:25:41,553 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6c1e23082935a9ab78c6a0d7f447d65, ASSIGN}] 2024-11-16T10:25:41,554 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6c1e23082935a9ab78c6a0d7f447d65, ASSIGN 2024-11-16T10:25:41,555 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6c1e23082935a9ab78c6a0d7f447d65, ASSIGN; state=OFFLINE, location=b16ff38458c1,35873,1731752627506; forceNewPlan=false, retain=false 2024-11-16T10:25:41,706 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=b6c1e23082935a9ab78c6a0d7f447d65, regionState=OPENING, regionLocation=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:41,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6c1e23082935a9ab78c6a0d7f447d65, ASSIGN because future has completed 2024-11-16T10:25:41,711 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506}] 2024-11-16T10:25:41,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-16T10:25:41,873 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:41,873 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7752): Opening region: {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} 2024-11-16T10:25:41,874 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:41,875 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:25:41,875 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7794): checking encryption for b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:41,875 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7797): checking classloading for b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:41,878 INFO [StoreOpener-b6c1e23082935a9ab78c6a0d7f447d65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:41,881 INFO [StoreOpener-b6c1e23082935a9ab78c6a0d7f447d65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6c1e23082935a9ab78c6a0d7f447d65 columnFamilyName A 2024-11-16T10:25:41,881 DEBUG [StoreOpener-b6c1e23082935a9ab78c6a0d7f447d65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:41,881 INFO [StoreOpener-b6c1e23082935a9ab78c6a0d7f447d65-1 {}] regionserver.HStore(327): Store=b6c1e23082935a9ab78c6a0d7f447d65/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:25:41,881 INFO [StoreOpener-b6c1e23082935a9ab78c6a0d7f447d65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:41,883 INFO [StoreOpener-b6c1e23082935a9ab78c6a0d7f447d65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6c1e23082935a9ab78c6a0d7f447d65 columnFamilyName B 2024-11-16T10:25:41,883 DEBUG [StoreOpener-b6c1e23082935a9ab78c6a0d7f447d65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:41,883 INFO [StoreOpener-b6c1e23082935a9ab78c6a0d7f447d65-1 {}] regionserver.HStore(327): Store=b6c1e23082935a9ab78c6a0d7f447d65/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:25:41,883 INFO [StoreOpener-b6c1e23082935a9ab78c6a0d7f447d65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:41,884 INFO [StoreOpener-b6c1e23082935a9ab78c6a0d7f447d65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6c1e23082935a9ab78c6a0d7f447d65 columnFamilyName C 2024-11-16T10:25:41,885 DEBUG [StoreOpener-b6c1e23082935a9ab78c6a0d7f447d65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:25:41,885 INFO [StoreOpener-b6c1e23082935a9ab78c6a0d7f447d65-1 {}] regionserver.HStore(327): Store=b6c1e23082935a9ab78c6a0d7f447d65/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:25:41,885 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1038): replaying wal for b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:41,886 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:41,886 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:41,887 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1048): stopping wal replay for b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:41,888 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1060): Cleaning up temporary data for b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:41,888 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T10:25:41,889 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1093): writing seq id for b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:41,891 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T10:25:41,892 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1114): Opened b6c1e23082935a9ab78c6a0d7f447d65; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66019820, jitterRate=-0.016228020191192627}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T10:25:41,892 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:41,893 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1006): Region open journal for b6c1e23082935a9ab78c6a0d7f447d65: Running coprocessor pre-open hook at 1731752741875Writing region info on filesystem at 1731752741875Initializing all the Stores at 1731752741877 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752741877Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752741878 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752741878Cleaning up temporary data from old regions at 1731752741888 (+10 ms)Running coprocessor post-open hooks at 1731752741892 (+4 ms)Region opened successfully at 1731752741893 (+1 ms) 2024-11-16T10:25:41,894 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., pid=114, masterSystemTime=1731752741865 2024-11-16T10:25:41,896 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:41,896 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:41,897 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=b6c1e23082935a9ab78c6a0d7f447d65, regionState=OPEN, openSeqNum=2, regionLocation=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:41,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 because future has completed 2024-11-16T10:25:41,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-16T10:25:41,902 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; OpenRegionProcedure b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 in 188 msec 2024-11-16T10:25:41,904 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-16T10:25:41,904 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6c1e23082935a9ab78c6a0d7f447d65, ASSIGN in 349 msec 2024-11-16T10:25:41,905 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T10:25:41,905 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752741905"}]},"ts":"1731752741905"} 2024-11-16T10:25:41,907 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T10:25:41,908 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T10:25:41,910 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2220 sec 2024-11-16T10:25:42,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-16T10:25:42,838 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:25:42,841 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72240ede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,841 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:25:42,841 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:25:42,842 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:25:42,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:25:42,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:25:42,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1108c6ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:25:42,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:25:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:42,845 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:25:42,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aef6ab4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,846 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:25:42,848 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:25:42,848 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:25:42,849 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:25:42,850 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T10:25:42,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is b16ff38458c1,38809,1731752626047 2024-11-16T10:25:42,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@33793453 2024-11-16T10:25:42,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T10:25:42,852 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T10:25:42,853 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503e1f3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,853 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:25:42,854 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:25:42,854 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:25:42,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:25:42,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:25:42,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17317a64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:25:42,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:25:42,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:42,856 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:25:42,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43a9d589, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,857 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205b33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,857 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:25:42,857 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:25:42,858 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:25:42,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:25:42,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:25:42,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10bbfd12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:25:42,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:25:42,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:42,859 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:25:42,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43f754e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e100fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,860 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:25:42,860 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:25:42,861 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:25:42,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:25:42,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:25:42,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e7428b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:25:42,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:25:42,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:42,862 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:25:42,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@542b5eb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,862 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e40afb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,862 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:25:42,862 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:25:42,863 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:25:42,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:25:42,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:25:42,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@480d8ba1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:25:42,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:25:42,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:42,864 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:25:42,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f17adf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,865 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9f4da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,865 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:25:42,865 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:25:42,865 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:25:42,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:25:42,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:25:42,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4898a38c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:25:42,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:25:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:42,866 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:25:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@224889ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,867 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4cbf6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,867 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:25:42,867 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:25:42,867 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:25:42,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:25:42,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:25:42,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ef47284, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:25:42,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:25:42,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:42,869 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:25:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a6ac31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,869 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83334c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,869 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:25:42,870 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:25:42,870 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:25:42,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:25:42,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:25:42,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7be7ccf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:25:42,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:25:42,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:42,871 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:25:42,872 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13a66d58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,872 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3532b5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,872 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:25:42,873 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:25:42,873 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:25:42,873 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:25:42,873 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:25:42,873 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@384f0ea1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,873 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:25:42,874 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:25:42,874 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:42,874 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:25:42,875 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c504493, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,876 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3088fab0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,876 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:25:42,876 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:25:42,876 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:25:42,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:25:42,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:25:42,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f210748, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:25:42,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:25:42,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:42,878 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:25:42,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e5379a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,879 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@663bb5d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,879 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:25:42,879 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:25:42,879 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:25:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:25:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:25:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32d3cfeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:25:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:25:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:25:42,881 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:25:42,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@231c0870, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:25:42,882 DEBUG [Thread-1797 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:25:42,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:25:42,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:25:42,883 DEBUG [Thread-1782 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:25:42,883 DEBUG [Thread-1793 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:25:42,884 DEBUG [Thread-1784 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:25:42,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:25:42,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:25:42,884 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:25:42,884 DEBUG [Thread-1786 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:25:42,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:25:42,885 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:25:42,885 DEBUG [Thread-1799 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:25:42,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:25:42,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:25:42,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:25:42,886 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:25:42,886 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:25:42,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:25:42,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2] 2024-11-16T10:25:42,887 DEBUG [Thread-1795 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:25:42,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:25:42,887 DEBUG [Thread-1788 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:25:42,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:25:42,887 DEBUG [Thread-1790 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:25:42,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2] 2024-11-16T10:25:42,888 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:25:42,888 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38866, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:25:42,888 DEBUG [Thread-1795 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:25:42,888 DEBUG [Thread-1795 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:25:42,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2] 2024-11-16T10:25:42,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2] 2024-11-16T10:25:42,889 DEBUG [Thread-1790 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:25:42,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2] 2024-11-16T10:25:42,889 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:25:42,890 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:25:42,890 DEBUG [Thread-1801 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:25:42,891 DEBUG [Thread-1790 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:25:42,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-16T10:25:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2] 2024-11-16T10:25:42,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:25:42,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:25:42,893 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:25:42,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T10:25:42,894 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:25:42,894 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:25:42,894 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:25:42,894 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:25:42,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:25:42,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2] 2024-11-16T10:25:42,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2] 2024-11-16T10:25:42,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2] 2024-11-16T10:25:42,902 DEBUG [Thread-1788 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:25:42,902 DEBUG [Thread-1788 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:25:42,903 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:25:42,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6c1e23082935a9ab78c6a0d7f447d65 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T10:25:42,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:42,904 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:25:42,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2] 2024-11-16T10:25:42,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/4d2d08317a36443cb841d33239f4b568 is 50, key is test_row_0/A:col10/1731752742901/Put/seqid=0 2024-11-16T10:25:42,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742216_1392 (size=14341) 2024-11-16T10:25:42,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/4d2d08317a36443cb841d33239f4b568 2024-11-16T10:25:42,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:42,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752802934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:42,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:42,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752802934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:42,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:42,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:42,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:42,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:42,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:42,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:42,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:42,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752802935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:42,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:42,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:42,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:42,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:42,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752802943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:42,945 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:42,945 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:42,945 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:42,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/d54805739bff4832a7ddaf6720ae272f is 50, key is test_row_0/B:col10/1731752742901/Put/seqid=0 2024-11-16T10:25:42,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752802945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:42,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:42,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:42,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:42,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742217_1393 (size=12001) 2024-11-16T10:25:42,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T10:25:43,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-16T10:25:43,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:43,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:43,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:43,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:43,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:43,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:43,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:43,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752803046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:43,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:43,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752803046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:43,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:43,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:43,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:43,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752803056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:43,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:43,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:43,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752803056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:43,062 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,062 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,062 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:43,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:43,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752803067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:43,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-16T10:25:43,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:43,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:43,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:43,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:43,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:43,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:43,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T10:25:43,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:43,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752803256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:43,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:43,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:43,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752803256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:43,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:43,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:43,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752803266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:43,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:43,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:43,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752803266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:43,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:43,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:43,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752803276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:43,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:43,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-16T10:25:43,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:43,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:43,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:43,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:43,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:43,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:43,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/d54805739bff4832a7ddaf6720ae272f 2024-11-16T10:25:43,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/e5a15a61248f450081d859f9311a2381 is 50, key is test_row_0/C:col10/1731752742901/Put/seqid=0 2024-11-16T10:25:43,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742218_1394 (size=12001) 2024-11-16T10:25:43,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-16T10:25:43,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:43,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:43,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:43,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:43,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:43,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:43,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T10:25:43,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:43,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752803566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:43,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:43,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752803566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:43,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:43,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:43,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:43,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752803576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:43,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:43,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:43,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752803576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:43,582 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,582 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,582 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:43,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:43,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752803586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:43,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-16T10:25:43,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:43,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:43,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:43,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:43,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:43,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:43,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/e5a15a61248f450081d859f9311a2381 2024-11-16T10:25:43,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/4d2d08317a36443cb841d33239f4b568 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/4d2d08317a36443cb841d33239f4b568 2024-11-16T10:25:43,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/4d2d08317a36443cb841d33239f4b568, entries=200, sequenceid=14, filesize=14.0 K 2024-11-16T10:25:43,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/d54805739bff4832a7ddaf6720ae272f as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/d54805739bff4832a7ddaf6720ae272f 2024-11-16T10:25:43,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/d54805739bff4832a7ddaf6720ae272f, entries=150, sequenceid=14, filesize=11.7 K 2024-11-16T10:25:43,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/e5a15a61248f450081d859f9311a2381 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/e5a15a61248f450081d859f9311a2381 2024-11-16T10:25:43,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/e5a15a61248f450081d859f9311a2381, entries=150, sequenceid=14, filesize=11.7 K 2024-11-16T10:25:43,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for b6c1e23082935a9ab78c6a0d7f447d65 in 888ms, sequenceid=14, compaction requested=false 2024-11-16T10:25:43,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:43,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-16T10:25:43,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:43,811 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing b6c1e23082935a9ab78c6a0d7f447d65 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T10:25:43,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/bfa5dd551f0d4b4fb131d0fcd51b93a7 is 50, key is test_row_0/A:col10/1731752742933/Put/seqid=0 2024-11-16T10:25:43,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742219_1395 (size=12001) 2024-11-16T10:25:44,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T10:25:44,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:44,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:44,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:44,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752804108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:44,111 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,111 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:44,111 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:44,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752804108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:44,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:44,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:44,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752804109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:44,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:44,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752804110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:44,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:44,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:44,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:44,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752804111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:44,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:44,221 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/bfa5dd551f0d4b4fb131d0fcd51b93a7 2024-11-16T10:25:44,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752804216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:44,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:44,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:44,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752804216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:44,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:44,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/fefb5b5ba8154afa91ead4febeccfc76 is 50, key is test_row_0/B:col10/1731752742933/Put/seqid=0 2024-11-16T10:25:44,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:44,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752804226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:44,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:44,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752804228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:44,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:44,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:44,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752804228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:44,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:44,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:44,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742220_1396 (size=12001) 2024-11-16T10:25:44,241 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/fefb5b5ba8154afa91ead4febeccfc76 2024-11-16T10:25:44,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/a756f068737249cc846ef4820a88f38a is 50, key is test_row_0/C:col10/1731752742933/Put/seqid=0 2024-11-16T10:25:44,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742221_1397 (size=12001) 2024-11-16T10:25:44,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752804427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:44,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752804427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:44,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752804436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:44,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752804436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:44,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752804436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:44,651 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/a756f068737249cc846ef4820a88f38a 2024-11-16T10:25:44,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/bfa5dd551f0d4b4fb131d0fcd51b93a7 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/bfa5dd551f0d4b4fb131d0fcd51b93a7 2024-11-16T10:25:44,657 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/bfa5dd551f0d4b4fb131d0fcd51b93a7, entries=150, sequenceid=38, filesize=11.7 K 2024-11-16T10:25:44,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/fefb5b5ba8154afa91ead4febeccfc76 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/fefb5b5ba8154afa91ead4febeccfc76 2024-11-16T10:25:44,661 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/fefb5b5ba8154afa91ead4febeccfc76, entries=150, sequenceid=38, filesize=11.7 K 2024-11-16T10:25:44,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/a756f068737249cc846ef4820a88f38a as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/a756f068737249cc846ef4820a88f38a 2024-11-16T10:25:44,665 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/a756f068737249cc846ef4820a88f38a, entries=150, sequenceid=38, filesize=11.7 K 2024-11-16T10:25:44,666 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for b6c1e23082935a9ab78c6a0d7f447d65 in 855ms, sequenceid=38, compaction requested=false 2024-11-16T10:25:44,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:44,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:44,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-16T10:25:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-16T10:25:44,668 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-16T10:25:44,668 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7730 sec 2024-11-16T10:25:44,670 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.7790 sec 2024-11-16T10:25:44,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:44,738 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:25:44,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6c1e23082935a9ab78c6a0d7f447d65 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T10:25:44,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/276e5ad7fcee4f40aced09e635eb61b9 is 50, key is test_row_0/A:col10/1731752744107/Put/seqid=0 2024-11-16T10:25:44,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742222_1398 (size=16681) 2024-11-16T10:25:44,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:44,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752804773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:44,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:44,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:44,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752804774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:44,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:44,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:44,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752804774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:44,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:44,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752804775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:44,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:44,784 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,784 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,784 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:44,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:44,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752804780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:44,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:44,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:44,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752804886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:44,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752804886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:44,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752804886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:44,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752804887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:44,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:44,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:44,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:44,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752804896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:44,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:44,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:45,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T10:25:45,036 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:25:45,037 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:25:45,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-16T10:25:45,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T10:25:45,038 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:25:45,039 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:25:45,039 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:25:45,096 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T10:25:45,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:45,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752805097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:45,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:45,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752805097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:45,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:45,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:45,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752805097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:45,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:45,099 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,099 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,100 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:45,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752805106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:45,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:45,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752805106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:45,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:45,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T10:25:45,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/276e5ad7fcee4f40aced09e635eb61b9 2024-11-16T10:25:45,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/4574249379e74bf887341d938a344dfe is 50, key is test_row_0/B:col10/1731752744107/Put/seqid=0 2024-11-16T10:25:45,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742223_1399 (size=12001) 2024-11-16T10:25:45,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T10:25:45,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:45,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:45,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:45,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:45,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:45,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:45,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T10:25:45,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:45,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:45,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:45,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:45,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:45,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:45,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T10:25:45,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752805407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:45,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:45,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752805407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:45,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:45,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:45,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752805407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:45,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:45,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:45,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:45,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752805416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:45,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:45,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752805416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:45,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:45,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:45,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T10:25:45,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:45,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:45,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:45,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:45,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:45,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:45,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/4574249379e74bf887341d938a344dfe 2024-11-16T10:25:45,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/142a67c7424e475a89e9dc832bc3808a is 50, key is test_row_0/C:col10/1731752744107/Put/seqid=0 2024-11-16T10:25:45,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742224_1400 (size=12001) 2024-11-16T10:25:45,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/142a67c7424e475a89e9dc832bc3808a 2024-11-16T10:25:45,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/276e5ad7fcee4f40aced09e635eb61b9 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/276e5ad7fcee4f40aced09e635eb61b9 2024-11-16T10:25:45,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/276e5ad7fcee4f40aced09e635eb61b9, entries=250, sequenceid=52, filesize=16.3 K 2024-11-16T10:25:45,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/4574249379e74bf887341d938a344dfe as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/4574249379e74bf887341d938a344dfe 2024-11-16T10:25:45,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/4574249379e74bf887341d938a344dfe, entries=150, sequenceid=52, filesize=11.7 K 2024-11-16T10:25:45,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/142a67c7424e475a89e9dc832bc3808a as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/142a67c7424e475a89e9dc832bc3808a 2024-11-16T10:25:45,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/142a67c7424e475a89e9dc832bc3808a, entries=150, sequenceid=52, filesize=11.7 K 2024-11-16T10:25:45,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for b6c1e23082935a9ab78c6a0d7f447d65 in 844ms, sequenceid=52, compaction requested=true 2024-11-16T10:25:45,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:45,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6c1e23082935a9ab78c6a0d7f447d65:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:25:45,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:45,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6c1e23082935a9ab78c6a0d7f447d65:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:25:45,583 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:25:45,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:45,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6c1e23082935a9ab78c6a0d7f447d65:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:25:45,583 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:25:45,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:45,584 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:25:45,584 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:25:45,584 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): b6c1e23082935a9ab78c6a0d7f447d65/B is initiating minor compaction (all files) 2024-11-16T10:25:45,584 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): b6c1e23082935a9ab78c6a0d7f447d65/A is initiating minor compaction (all files) 2024-11-16T10:25:45,584 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6c1e23082935a9ab78c6a0d7f447d65/A in TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:45,584 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6c1e23082935a9ab78c6a0d7f447d65/B in TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:45,584 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/4d2d08317a36443cb841d33239f4b568, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/bfa5dd551f0d4b4fb131d0fcd51b93a7, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/276e5ad7fcee4f40aced09e635eb61b9] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp, totalSize=42.0 K 2024-11-16T10:25:45,584 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/d54805739bff4832a7ddaf6720ae272f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/fefb5b5ba8154afa91ead4febeccfc76, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/4574249379e74bf887341d938a344dfe] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp, totalSize=35.2 K 2024-11-16T10:25:45,585 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting d54805739bff4832a7ddaf6720ae272f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731752742896 2024-11-16T10:25:45,585 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d2d08317a36443cb841d33239f4b568, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731752742896 2024-11-16T10:25:45,585 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfa5dd551f0d4b4fb131d0fcd51b93a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731752742920 2024-11-16T10:25:45,585 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting fefb5b5ba8154afa91ead4febeccfc76, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731752742920 2024-11-16T10:25:45,585 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 276e5ad7fcee4f40aced09e635eb61b9, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731752744107 2024-11-16T10:25:45,585 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 4574249379e74bf887341d938a344dfe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731752744107 2024-11-16T10:25:45,592 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6c1e23082935a9ab78c6a0d7f447d65#A#compaction#335 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:45,593 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/7c0f6b013c3a499c82a40438df23d0eb is 50, key is test_row_0/A:col10/1731752744107/Put/seqid=0 2024-11-16T10:25:45,594 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6c1e23082935a9ab78c6a0d7f447d65#B#compaction#336 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:45,594 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/b82206b610474c5a81c16f7b1cf3e5da is 50, key is test_row_0/B:col10/1731752744107/Put/seqid=0 2024-11-16T10:25:45,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742225_1401 (size=12104) 2024-11-16T10:25:45,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742226_1402 (size=12104) 2024-11-16T10:25:45,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T10:25:45,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:45,648 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing b6c1e23082935a9ab78c6a0d7f447d65 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-16T10:25:45,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/7652b8dd75f34a42b1f49d2fd68a2628 is 50, key is test_row_0/A:col10/1731752744774/Put/seqid=0 2024-11-16T10:25:45,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742227_1403 (size=12001) 2024-11-16T10:25:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T10:25:45,829 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T10:25:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:45,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:45,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752805936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:45,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752805937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:45,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752805945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:45,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752805945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:45,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:45,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752805945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:45,955 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,955 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:45,955 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:46,003 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/7c0f6b013c3a499c82a40438df23d0eb as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/7c0f6b013c3a499c82a40438df23d0eb 2024-11-16T10:25:46,003 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/b82206b610474c5a81c16f7b1cf3e5da as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/b82206b610474c5a81c16f7b1cf3e5da 2024-11-16T10:25:46,007 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6c1e23082935a9ab78c6a0d7f447d65/A of b6c1e23082935a9ab78c6a0d7f447d65 into 7c0f6b013c3a499c82a40438df23d0eb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:46,007 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:46,007 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6c1e23082935a9ab78c6a0d7f447d65/B of b6c1e23082935a9ab78c6a0d7f447d65 into b82206b610474c5a81c16f7b1cf3e5da(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:46,007 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., storeName=b6c1e23082935a9ab78c6a0d7f447d65/A, priority=13, startTime=1731752745582; duration=0sec 2024-11-16T10:25:46,007 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:46,007 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., storeName=b6c1e23082935a9ab78c6a0d7f447d65/B, priority=13, startTime=1731752745583; duration=0sec 2024-11-16T10:25:46,007 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:46,007 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6c1e23082935a9ab78c6a0d7f447d65:A 2024-11-16T10:25:46,007 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:46,007 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6c1e23082935a9ab78c6a0d7f447d65:B 2024-11-16T10:25:46,007 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:25:46,008 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:25:46,008 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): b6c1e23082935a9ab78c6a0d7f447d65/C is initiating minor compaction (all files) 2024-11-16T10:25:46,008 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6c1e23082935a9ab78c6a0d7f447d65/C in TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:46,008 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/e5a15a61248f450081d859f9311a2381, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/a756f068737249cc846ef4820a88f38a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/142a67c7424e475a89e9dc832bc3808a] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp, totalSize=35.2 K 2024-11-16T10:25:46,008 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5a15a61248f450081d859f9311a2381, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731752742896 2024-11-16T10:25:46,008 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting a756f068737249cc846ef4820a88f38a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731752742920 2024-11-16T10:25:46,009 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 142a67c7424e475a89e9dc832bc3808a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731752744107 2024-11-16T10:25:46,014 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6c1e23082935a9ab78c6a0d7f447d65#C#compaction#338 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:46,015 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/d89a904932a1470f90be033faf04ade8 is 50, key is test_row_0/C:col10/1731752744107/Put/seqid=0 2024-11-16T10:25:46,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742228_1404 (size=12104) 2024-11-16T10:25:46,057 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/7652b8dd75f34a42b1f49d2fd68a2628 2024-11-16T10:25:46,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:46,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752806056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:46,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:46,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:46,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752806056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:46,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:46,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:46,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752806056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:46,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:46,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:46,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752806056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:46,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:46,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/5d51d95ac8224d5aa9743c1709fc4ff6 is 50, key is test_row_0/B:col10/1731752744774/Put/seqid=0 2024-11-16T10:25:46,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742229_1405 (size=12001) 2024-11-16T10:25:46,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:46,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752806067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:46,073 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,073 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,073 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:46,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T10:25:46,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:46,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752806266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:46,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752806266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752806266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:46,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:46,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752806276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:46,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:46,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752806277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:46,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:46,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:46,426 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/d89a904932a1470f90be033faf04ade8 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/d89a904932a1470f90be033faf04ade8 2024-11-16T10:25:46,430 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6c1e23082935a9ab78c6a0d7f447d65/C of b6c1e23082935a9ab78c6a0d7f447d65 into d89a904932a1470f90be033faf04ade8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:46,430 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:46,430 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., storeName=b6c1e23082935a9ab78c6a0d7f447d65/C, priority=13, startTime=1731752745583; duration=0sec 2024-11-16T10:25:46,430 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:46,430 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6c1e23082935a9ab78c6a0d7f447d65:C 2024-11-16T10:25:46,468 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/5d51d95ac8224d5aa9743c1709fc4ff6 2024-11-16T10:25:46,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/730e2c33620e4ae592b55f7e10135623 is 50, key is test_row_0/C:col10/1731752744774/Put/seqid=0 2024-11-16T10:25:46,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742230_1406 (size=12001) 2024-11-16T10:25:46,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:46,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752806576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:46,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:46,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752806576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:46,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:46,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:46,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:46,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752806586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:46,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:46,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:46,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752806586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:46,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:46,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:46,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752806586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:46,591 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,591 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:46,591 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:46,878 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/730e2c33620e4ae592b55f7e10135623 2024-11-16T10:25:46,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/7652b8dd75f34a42b1f49d2fd68a2628 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/7652b8dd75f34a42b1f49d2fd68a2628 2024-11-16T10:25:46,884 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/7652b8dd75f34a42b1f49d2fd68a2628, entries=150, sequenceid=74, filesize=11.7 K 2024-11-16T10:25:46,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/5d51d95ac8224d5aa9743c1709fc4ff6 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/5d51d95ac8224d5aa9743c1709fc4ff6 2024-11-16T10:25:46,888 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/5d51d95ac8224d5aa9743c1709fc4ff6, entries=150, sequenceid=74, filesize=11.7 K 2024-11-16T10:25:46,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/730e2c33620e4ae592b55f7e10135623 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/730e2c33620e4ae592b55f7e10135623 2024-11-16T10:25:46,891 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/730e2c33620e4ae592b55f7e10135623, entries=150, sequenceid=74, filesize=11.7 K 2024-11-16T10:25:46,892 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for b6c1e23082935a9ab78c6a0d7f447d65 in 1244ms, sequenceid=74, compaction requested=false 2024-11-16T10:25:46,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:46,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:46,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-16T10:25:46,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-16T10:25:46,894 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-16T10:25:46,894 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8540 sec 2024-11-16T10:25:46,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.8580 sec 2024-11-16T10:25:47,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:47,090 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:25:47,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6c1e23082935a9ab78c6a0d7f447d65 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-16T10:25:47,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/15993e13a992499080f09975714e2628 is 50, key is test_row_0/A:col10/1731752747089/Put/seqid=0 2024-11-16T10:25:47,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742231_1407 (size=14341) 2024-11-16T10:25:47,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:47,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:47,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752807124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:47,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752807123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:47,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:47,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:47,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:47,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752807155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:47,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:47,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:47,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752807156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:47,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:47,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:47,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752807156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:47,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:47,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T10:25:47,186 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:25:47,187 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:25:47,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-16T10:25:47,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T10:25:47,188 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:25:47,188 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:25:47,189 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:25:47,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:47,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752807266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:47,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:47,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752807266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:47,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:47,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:47,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752807276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:47,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:47,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:47,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752807276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:47,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:47,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752807276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:47,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:47,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:47,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T10:25:47,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T10:25:47,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:47,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:47,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:47,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:47,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:47,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:47,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:47,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752807486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:47,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752807486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:47,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T10:25:47,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:47,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:47,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:47,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:47,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:47,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:47,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752807496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:47,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/15993e13a992499080f09975714e2628 2024-11-16T10:25:47,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:47,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:47,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752807496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:47,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:47,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:47,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752807496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:47,501 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,501 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,501 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:47,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/b1944de795d440a79b979f55449f21cf is 50, key is test_row_0/B:col10/1731752747089/Put/seqid=0 2024-11-16T10:25:47,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T10:25:47,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742232_1408 (size=12001) 2024-11-16T10:25:47,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T10:25:47,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:47,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:47,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:47,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:47,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:47,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752807796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:47,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:47,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752807796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:47,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:47,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T10:25:47,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:47,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:47,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:47,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:47,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:47,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752807806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:47,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:47,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752807806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:47,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:47,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:47,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752807806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:47,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:47,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:47,814 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T10:25:47,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/b1944de795d440a79b979f55449f21cf 2024-11-16T10:25:47,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/b9c7db1913094d6c8716112fc8e7d0ea is 50, key is test_row_0/C:col10/1731752747089/Put/seqid=0 2024-11-16T10:25:47,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742233_1409 (size=12001) 2024-11-16T10:25:47,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T10:25:47,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:47,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:47,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:47,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:47,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:48,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T10:25:48,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:48,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:48,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:48,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:48,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:48,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:48,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T10:25:48,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:48,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:48,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:48,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:48,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:48,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:48,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752808306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:48,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:48,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:48,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:48,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:48,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752808306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:48,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:48,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:48,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:48,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/b9c7db1913094d6c8716112fc8e7d0ea 2024-11-16T10:25:48,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/15993e13a992499080f09975714e2628 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/15993e13a992499080f09975714e2628 2024-11-16T10:25:48,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/15993e13a992499080f09975714e2628, entries=200, sequenceid=93, filesize=14.0 K 2024-11-16T10:25:48,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/b1944de795d440a79b979f55449f21cf as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/b1944de795d440a79b979f55449f21cf 2024-11-16T10:25:48,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/b1944de795d440a79b979f55449f21cf, entries=150, sequenceid=93, filesize=11.7 K 2024-11-16T10:25:48,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/b9c7db1913094d6c8716112fc8e7d0ea as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/b9c7db1913094d6c8716112fc8e7d0ea 2024-11-16T10:25:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T10:25:48,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/b9c7db1913094d6c8716112fc8e7d0ea, entries=150, sequenceid=93, filesize=11.7 K 2024-11-16T10:25:48,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=127.47 KB/130530 for b6c1e23082935a9ab78c6a0d7f447d65 in 1241ms, sequenceid=93, compaction requested=true 2024-11-16T10:25:48,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:48,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6c1e23082935a9ab78c6a0d7f447d65:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:25:48,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:48,331 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:25:48,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6c1e23082935a9ab78c6a0d7f447d65:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:25:48,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:48,331 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:25:48,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6c1e23082935a9ab78c6a0d7f447d65:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:25:48,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:48,333 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:25:48,333 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:25:48,333 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): b6c1e23082935a9ab78c6a0d7f447d65/A is initiating minor compaction (all files) 2024-11-16T10:25:48,333 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): b6c1e23082935a9ab78c6a0d7f447d65/B is initiating minor compaction (all files) 2024-11-16T10:25:48,333 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6c1e23082935a9ab78c6a0d7f447d65/B in TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:48,333 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6c1e23082935a9ab78c6a0d7f447d65/A in TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:48,333 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/b82206b610474c5a81c16f7b1cf3e5da, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/5d51d95ac8224d5aa9743c1709fc4ff6, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/b1944de795d440a79b979f55449f21cf] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp, totalSize=35.3 K 2024-11-16T10:25:48,333 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/7c0f6b013c3a499c82a40438df23d0eb, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/7652b8dd75f34a42b1f49d2fd68a2628, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/15993e13a992499080f09975714e2628] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp, totalSize=37.5 K 2024-11-16T10:25:48,334 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c0f6b013c3a499c82a40438df23d0eb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731752744107 2024-11-16T10:25:48,334 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting b82206b610474c5a81c16f7b1cf3e5da, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731752744107 2024-11-16T10:25:48,334 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7652b8dd75f34a42b1f49d2fd68a2628, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731752744772 2024-11-16T10:25:48,334 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d51d95ac8224d5aa9743c1709fc4ff6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731752744772 2024-11-16T10:25:48,334 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15993e13a992499080f09975714e2628, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731752745943 2024-11-16T10:25:48,334 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting b1944de795d440a79b979f55449f21cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731752745943 2024-11-16T10:25:48,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:48,334 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:25:48,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6c1e23082935a9ab78c6a0d7f447d65 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T10:25:48,339 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6c1e23082935a9ab78c6a0d7f447d65#B#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:48,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/28a98b91d4e4458db38252388a992403 is 50, key is test_row_0/A:col10/1731752747132/Put/seqid=0 2024-11-16T10:25:48,339 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/4faa3b82741d481fae1a3d1d4b9b76f2 is 50, key is test_row_0/B:col10/1731752747089/Put/seqid=0 2024-11-16T10:25:48,341 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6c1e23082935a9ab78c6a0d7f447d65#A#compaction#346 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:48,342 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/e5830dcfe5cd4bd08b1c11eb36cc9a9e is 50, key is test_row_0/A:col10/1731752747089/Put/seqid=0 2024-11-16T10:25:48,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742234_1410 (size=14341) 2024-11-16T10:25:48,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/28a98b91d4e4458db38252388a992403 2024-11-16T10:25:48,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742236_1412 (size=12207) 2024-11-16T10:25:48,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/4882ca00a39544e2aebdcc5c05d4e786 is 50, key is test_row_0/B:col10/1731752747132/Put/seqid=0 2024-11-16T10:25:48,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742235_1411 (size=12207) 2024-11-16T10:25:48,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742237_1413 (size=12001) 2024-11-16T10:25:48,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/4882ca00a39544e2aebdcc5c05d4e786 2024-11-16T10:25:48,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/d72ee1f1696b42a28c5947a097baca17 is 50, key is test_row_0/C:col10/1731752747132/Put/seqid=0 2024-11-16T10:25:48,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742238_1414 (size=12001) 2024-11-16T10:25:48,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:48,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752808385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:48,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:48,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:48,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:48,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:48,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752808385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:48,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:48,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:48,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:48,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752808386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:48,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:48,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:48,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:48,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:48,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T10:25:48,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:48,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:48,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:48,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:48,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:48,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:48,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752808506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:48,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:48,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:48,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:48,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:48,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752808506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:48,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:48,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:48,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:48,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:48,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752808506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:48,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:48,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:48,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:48,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T10:25:48,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:48,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:48,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:48,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:48,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:48,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:48,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T10:25:48,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:48,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:48,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:48,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:48,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:48,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:48,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:48,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752808716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:48,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:48,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:48,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752808716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:48,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:48,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:48,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:48,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:48,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752808716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:48,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:48,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:48,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:48,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:48,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:48,762 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/e5830dcfe5cd4bd08b1c11eb36cc9a9e as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/e5830dcfe5cd4bd08b1c11eb36cc9a9e 2024-11-16T10:25:48,766 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6c1e23082935a9ab78c6a0d7f447d65/A of b6c1e23082935a9ab78c6a0d7f447d65 into e5830dcfe5cd4bd08b1c11eb36cc9a9e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:48,766 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:48,766 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., storeName=b6c1e23082935a9ab78c6a0d7f447d65/A, priority=13, startTime=1731752748331; duration=0sec 2024-11-16T10:25:48,766 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:48,766 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6c1e23082935a9ab78c6a0d7f447d65:A 2024-11-16T10:25:48,766 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:25:48,767 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:25:48,768 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): b6c1e23082935a9ab78c6a0d7f447d65/C is initiating minor compaction (all files) 2024-11-16T10:25:48,768 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6c1e23082935a9ab78c6a0d7f447d65/C in TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:48,768 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/d89a904932a1470f90be033faf04ade8, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/730e2c33620e4ae592b55f7e10135623, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/b9c7db1913094d6c8716112fc8e7d0ea] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp, totalSize=35.3 K 2024-11-16T10:25:48,768 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting d89a904932a1470f90be033faf04ade8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731752744107 2024-11-16T10:25:48,768 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 730e2c33620e4ae592b55f7e10135623, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731752744772 2024-11-16T10:25:48,769 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9c7db1913094d6c8716112fc8e7d0ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731752745943 2024-11-16T10:25:48,770 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/4faa3b82741d481fae1a3d1d4b9b76f2 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/4faa3b82741d481fae1a3d1d4b9b76f2 2024-11-16T10:25:48,774 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6c1e23082935a9ab78c6a0d7f447d65/B of b6c1e23082935a9ab78c6a0d7f447d65 into 4faa3b82741d481fae1a3d1d4b9b76f2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:48,774 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:48,774 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., storeName=b6c1e23082935a9ab78c6a0d7f447d65/B, priority=13, startTime=1731752748331; duration=0sec 2024-11-16T10:25:48,774 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:48,774 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6c1e23082935a9ab78c6a0d7f447d65:B 2024-11-16T10:25:48,775 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6c1e23082935a9ab78c6a0d7f447d65#C#compaction#349 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:48,775 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/057cdc5828664fc68e9af5f8e58ce4b8 is 50, key is test_row_0/C:col10/1731752747089/Put/seqid=0 2024-11-16T10:25:48,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742239_1415 (size=12207) 2024-11-16T10:25:48,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/d72ee1f1696b42a28c5947a097baca17 2024-11-16T10:25:48,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/28a98b91d4e4458db38252388a992403 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/28a98b91d4e4458db38252388a992403 2024-11-16T10:25:48,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/28a98b91d4e4458db38252388a992403, entries=200, sequenceid=116, filesize=14.0 K 2024-11-16T10:25:48,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/4882ca00a39544e2aebdcc5c05d4e786 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/4882ca00a39544e2aebdcc5c05d4e786 2024-11-16T10:25:48,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/4882ca00a39544e2aebdcc5c05d4e786, entries=150, sequenceid=116, filesize=11.7 K 2024-11-16T10:25:48,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/d72ee1f1696b42a28c5947a097baca17 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/d72ee1f1696b42a28c5947a097baca17 2024-11-16T10:25:48,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/d72ee1f1696b42a28c5947a097baca17, entries=150, sequenceid=116, filesize=11.7 K 2024-11-16T10:25:48,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for b6c1e23082935a9ab78c6a0d7f447d65 in 463ms, sequenceid=116, compaction requested=false 2024-11-16T10:25:48,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:48,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T10:25:48,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:48,867 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing b6c1e23082935a9ab78c6a0d7f447d65 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T10:25:48,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/8868d0fb8c924042b0d80a0be649cb63 is 50, key is test_row_0/A:col10/1731752748381/Put/seqid=0 2024-11-16T10:25:48,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742240_1416 (size=12001) 2024-11-16T10:25:49,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:49,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:49,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:49,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752809079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:49,088 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:49,088 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:49,088 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:49,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:49,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752809085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:49,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:49,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:49,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752809088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:49,182 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/057cdc5828664fc68e9af5f8e58ce4b8 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/057cdc5828664fc68e9af5f8e58ce4b8 2024-11-16T10:25:49,186 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6c1e23082935a9ab78c6a0d7f447d65/C of b6c1e23082935a9ab78c6a0d7f447d65 into 057cdc5828664fc68e9af5f8e58ce4b8(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:49,186 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:49,186 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., storeName=b6c1e23082935a9ab78c6a0d7f447d65/C, priority=13, startTime=1731752748331; duration=0sec 2024-11-16T10:25:49,186 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:49,186 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6c1e23082935a9ab78c6a0d7f447d65:C 2024-11-16T10:25:49,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:49,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752809196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:49,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:49,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752809196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:49,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:49,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752809206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:49,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:49,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:49,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:49,275 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/8868d0fb8c924042b0d80a0be649cb63 2024-11-16T10:25:49,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/b73e05e5cf38491da19e19e4d5a87f44 is 50, key is test_row_0/B:col10/1731752748381/Put/seqid=0 2024-11-16T10:25:49,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742241_1417 (size=12001) 2024-11-16T10:25:49,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:49,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752809316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:49,326 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_2' in b6c1e23082935a9ab78c6a0d7f447d65 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:25:49,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:49,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:49,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:49,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:49,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752809326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:49,332 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_1' in b6c1e23082935a9ab78c6a0d7f447d65 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:25:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:49,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T10:25:49,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:49,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752809406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:49,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:49,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:49,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752809407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:49,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:49,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:49,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:49,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:49,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:49,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:49,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752809416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:49,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:49,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:49,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:49,685 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/b73e05e5cf38491da19e19e4d5a87f44 2024-11-16T10:25:49,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/a466d20f7ac94ad6a0761339f83067c2 is 50, key is test_row_0/C:col10/1731752748381/Put/seqid=0 2024-11-16T10:25:49,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742242_1418 (size=12001) 2024-11-16T10:25:49,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752809716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:49,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:49,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:49,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:49,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752809716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:49,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:49,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:49,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:49,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:49,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752809726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:49,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:49,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:49,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:50,093 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/a466d20f7ac94ad6a0761339f83067c2 2024-11-16T10:25:50,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/8868d0fb8c924042b0d80a0be649cb63 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/8868d0fb8c924042b0d80a0be649cb63 2024-11-16T10:25:50,100 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/8868d0fb8c924042b0d80a0be649cb63, entries=150, sequenceid=131, filesize=11.7 K 2024-11-16T10:25:50,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/b73e05e5cf38491da19e19e4d5a87f44 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/b73e05e5cf38491da19e19e4d5a87f44 2024-11-16T10:25:50,104 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/b73e05e5cf38491da19e19e4d5a87f44, entries=150, sequenceid=131, filesize=11.7 K 2024-11-16T10:25:50,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/a466d20f7ac94ad6a0761339f83067c2 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/a466d20f7ac94ad6a0761339f83067c2 2024-11-16T10:25:50,108 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/a466d20f7ac94ad6a0761339f83067c2, entries=150, sequenceid=131, filesize=11.7 K 2024-11-16T10:25:50,108 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for b6c1e23082935a9ab78c6a0d7f447d65 in 1241ms, sequenceid=131, compaction requested=true 2024-11-16T10:25:50,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:50,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:50,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-16T10:25:50,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-16T10:25:50,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-16T10:25:50,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9200 sec 2024-11-16T10:25:50,112 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.9240 sec 2024-11-16T10:25:50,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:50,231 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:25:50,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6c1e23082935a9ab78c6a0d7f447d65 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T10:25:50,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/63731870bc11466580687784b0bca801 is 50, key is test_row_0/A:col10/1731752750230/Put/seqid=0 2024-11-16T10:25:50,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742243_1419 (size=14541) 2024-11-16T10:25:50,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/63731870bc11466580687784b0bca801 2024-11-16T10:25:50,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/53b91319707744ba87467950605277d9 is 50, key is test_row_0/B:col10/1731752750230/Put/seqid=0 2024-11-16T10:25:50,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742244_1420 (size=12151) 2024-11-16T10:25:50,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/53b91319707744ba87467950605277d9 2024-11-16T10:25:50,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/93deb7b49b944888a48ad611124af178 is 50, key is test_row_0/C:col10/1731752750230/Put/seqid=0 2024-11-16T10:25:50,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742245_1421 (size=12151) 2024-11-16T10:25:50,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:50,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752810260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:50,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:50,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:50,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:50,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:50,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752810260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:50,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:50,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:50,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:50,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:50,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752810270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:50,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:50,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:50,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:50,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:50,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752810376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:50,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:50,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752810376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:50,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:50,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752810376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:50,506 WARN [regionserver/b16ff38458c1:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 2, running: 1 2024-11-16T10:25:50,506 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/4d2d08317a36443cb841d33239f4b568, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/bfa5dd551f0d4b4fb131d0fcd51b93a7, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/276e5ad7fcee4f40aced09e635eb61b9, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/7c0f6b013c3a499c82a40438df23d0eb, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/7652b8dd75f34a42b1f49d2fd68a2628, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/15993e13a992499080f09975714e2628] to archive 2024-11-16T10:25:50,507 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:25:50,509 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/4d2d08317a36443cb841d33239f4b568 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/4d2d08317a36443cb841d33239f4b568 2024-11-16T10:25:50,509 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/bfa5dd551f0d4b4fb131d0fcd51b93a7 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/bfa5dd551f0d4b4fb131d0fcd51b93a7 2024-11-16T10:25:50,510 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/276e5ad7fcee4f40aced09e635eb61b9 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/276e5ad7fcee4f40aced09e635eb61b9 2024-11-16T10:25:50,511 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/7c0f6b013c3a499c82a40438df23d0eb to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/7c0f6b013c3a499c82a40438df23d0eb 2024-11-16T10:25:50,512 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/7652b8dd75f34a42b1f49d2fd68a2628 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/7652b8dd75f34a42b1f49d2fd68a2628 2024-11-16T10:25:50,513 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/15993e13a992499080f09975714e2628 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/15993e13a992499080f09975714e2628 2024-11-16T10:25:50,514 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/d54805739bff4832a7ddaf6720ae272f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/fefb5b5ba8154afa91ead4febeccfc76, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/b82206b610474c5a81c16f7b1cf3e5da, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/4574249379e74bf887341d938a344dfe, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/5d51d95ac8224d5aa9743c1709fc4ff6, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/b1944de795d440a79b979f55449f21cf] to archive 2024-11-16T10:25:50,514 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:25:50,515 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/d54805739bff4832a7ddaf6720ae272f to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/d54805739bff4832a7ddaf6720ae272f 2024-11-16T10:25:50,516 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/fefb5b5ba8154afa91ead4febeccfc76 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/fefb5b5ba8154afa91ead4febeccfc76 2024-11-16T10:25:50,517 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/b82206b610474c5a81c16f7b1cf3e5da to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/b82206b610474c5a81c16f7b1cf3e5da 2024-11-16T10:25:50,518 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/4574249379e74bf887341d938a344dfe to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/4574249379e74bf887341d938a344dfe 2024-11-16T10:25:50,518 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/5d51d95ac8224d5aa9743c1709fc4ff6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/5d51d95ac8224d5aa9743c1709fc4ff6 2024-11-16T10:25:50,519 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/b1944de795d440a79b979f55449f21cf to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/b1944de795d440a79b979f55449f21cf 2024-11-16T10:25:50,521 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/e5a15a61248f450081d859f9311a2381, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/a756f068737249cc846ef4820a88f38a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/d89a904932a1470f90be033faf04ade8, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/142a67c7424e475a89e9dc832bc3808a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/730e2c33620e4ae592b55f7e10135623, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/b9c7db1913094d6c8716112fc8e7d0ea] to archive 2024-11-16T10:25:50,521 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:25:50,522 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/e5a15a61248f450081d859f9311a2381 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/e5a15a61248f450081d859f9311a2381 2024-11-16T10:25:50,523 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/a756f068737249cc846ef4820a88f38a to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/a756f068737249cc846ef4820a88f38a 2024-11-16T10:25:50,524 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/d89a904932a1470f90be033faf04ade8 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/d89a904932a1470f90be033faf04ade8 2024-11-16T10:25:50,525 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/142a67c7424e475a89e9dc832bc3808a to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/142a67c7424e475a89e9dc832bc3808a 2024-11-16T10:25:50,525 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/730e2c33620e4ae592b55f7e10135623 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/730e2c33620e4ae592b55f7e10135623 2024-11-16T10:25:50,526 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b16ff38458c1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/b9c7db1913094d6c8716112fc8e7d0ea to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/b9c7db1913094d6c8716112fc8e7d0ea 2024-11-16T10:25:50,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:50,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752810586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:50,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:50,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752810586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:50,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:50,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752810586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:50,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:50,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:50,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:50,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/93deb7b49b944888a48ad611124af178 2024-11-16T10:25:50,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/63731870bc11466580687784b0bca801 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/63731870bc11466580687784b0bca801 2024-11-16T10:25:50,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/63731870bc11466580687784b0bca801, entries=200, sequenceid=156, filesize=14.2 K 2024-11-16T10:25:50,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/53b91319707744ba87467950605277d9 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/53b91319707744ba87467950605277d9 2024-11-16T10:25:50,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/53b91319707744ba87467950605277d9, entries=150, sequenceid=156, filesize=11.9 K 2024-11-16T10:25:50,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/93deb7b49b944888a48ad611124af178 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/93deb7b49b944888a48ad611124af178 2024-11-16T10:25:50,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/93deb7b49b944888a48ad611124af178, entries=150, sequenceid=156, filesize=11.9 K 2024-11-16T10:25:50,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for b6c1e23082935a9ab78c6a0d7f447d65 in 445ms, sequenceid=156, compaction requested=true 2024-11-16T10:25:50,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:50,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6c1e23082935a9ab78c6a0d7f447d65:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:25:50,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:50,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6c1e23082935a9ab78c6a0d7f447d65:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:25:50,676 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:25:50,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:50,676 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:25:50,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6c1e23082935a9ab78c6a0d7f447d65:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:25:50,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:50,677 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:25:50,677 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): b6c1e23082935a9ab78c6a0d7f447d65/A is initiating minor compaction (all files) 2024-11-16T10:25:50,677 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6c1e23082935a9ab78c6a0d7f447d65/A in TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:50,677 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/e5830dcfe5cd4bd08b1c11eb36cc9a9e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/28a98b91d4e4458db38252388a992403, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/8868d0fb8c924042b0d80a0be649cb63, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/63731870bc11466580687784b0bca801] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp, totalSize=51.8 K 2024-11-16T10:25:50,677 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:25:50,678 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): b6c1e23082935a9ab78c6a0d7f447d65/B is initiating minor compaction (all files) 2024-11-16T10:25:50,678 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6c1e23082935a9ab78c6a0d7f447d65/B in TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:50,678 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5830dcfe5cd4bd08b1c11eb36cc9a9e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731752745943 2024-11-16T10:25:50,678 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/4faa3b82741d481fae1a3d1d4b9b76f2, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/4882ca00a39544e2aebdcc5c05d4e786, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/b73e05e5cf38491da19e19e4d5a87f44, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/53b91319707744ba87467950605277d9] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp, totalSize=47.2 K 2024-11-16T10:25:50,678 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28a98b91d4e4458db38252388a992403, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731752747122 2024-11-16T10:25:50,678 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 4faa3b82741d481fae1a3d1d4b9b76f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731752745943 2024-11-16T10:25:50,678 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 4882ca00a39544e2aebdcc5c05d4e786, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731752747132 2024-11-16T10:25:50,679 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8868d0fb8c924042b0d80a0be649cb63, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731752748381 2024-11-16T10:25:50,679 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63731870bc11466580687784b0bca801, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731752749078 2024-11-16T10:25:50,679 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting b73e05e5cf38491da19e19e4d5a87f44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731752748381 2024-11-16T10:25:50,679 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 53b91319707744ba87467950605277d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731752749084 2024-11-16T10:25:50,686 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6c1e23082935a9ab78c6a0d7f447d65#B#compaction#357 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:50,686 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6c1e23082935a9ab78c6a0d7f447d65#A#compaction#356 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:50,687 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/15ab4e4587db4fb98c3a969b11b42f3d is 50, key is test_row_0/B:col10/1731752750230/Put/seqid=0 2024-11-16T10:25:50,687 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/87ccfd6286ea46f98065ad24083f16f4 is 50, key is test_row_0/A:col10/1731752750230/Put/seqid=0 2024-11-16T10:25:50,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742246_1422 (size=12289) 2024-11-16T10:25:50,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742247_1423 (size=12289) 2024-11-16T10:25:50,703 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/87ccfd6286ea46f98065ad24083f16f4 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/87ccfd6286ea46f98065ad24083f16f4 2024-11-16T10:25:50,707 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6c1e23082935a9ab78c6a0d7f447d65/A of b6c1e23082935a9ab78c6a0d7f447d65 into 87ccfd6286ea46f98065ad24083f16f4(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:50,707 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:50,707 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., storeName=b6c1e23082935a9ab78c6a0d7f447d65/A, priority=12, startTime=1731752750676; duration=0sec 2024-11-16T10:25:50,707 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:50,707 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6c1e23082935a9ab78c6a0d7f447d65:A 2024-11-16T10:25:50,708 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:25:50,708 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:25:50,709 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): b6c1e23082935a9ab78c6a0d7f447d65/C is initiating minor compaction (all files) 2024-11-16T10:25:50,709 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6c1e23082935a9ab78c6a0d7f447d65/C in TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:50,709 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/057cdc5828664fc68e9af5f8e58ce4b8, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/d72ee1f1696b42a28c5947a097baca17, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/a466d20f7ac94ad6a0761339f83067c2, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/93deb7b49b944888a48ad611124af178] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp, totalSize=47.2 K 2024-11-16T10:25:50,709 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 057cdc5828664fc68e9af5f8e58ce4b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731752745943 2024-11-16T10:25:50,709 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting d72ee1f1696b42a28c5947a097baca17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731752747132 2024-11-16T10:25:50,709 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting a466d20f7ac94ad6a0761339f83067c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731752748381 2024-11-16T10:25:50,710 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93deb7b49b944888a48ad611124af178, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731752749084 2024-11-16T10:25:50,718 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6c1e23082935a9ab78c6a0d7f447d65#C#compaction#358 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:50,718 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/2b7f53029c3a4d8db4ce30a7bac4326a is 50, key is test_row_0/C:col10/1731752750230/Put/seqid=0 2024-11-16T10:25:50,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742248_1424 (size=12289) 2024-11-16T10:25:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:50,908 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:25:50,908 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6c1e23082935a9ab78c6a0d7f447d65 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T10:25:50,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/46f50474e257447abb583af4e987dc0e is 50, key is test_row_0/A:col10/1731752750269/Put/seqid=0 2024-11-16T10:25:50,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742249_1425 (size=14541) 2024-11-16T10:25:50,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752810965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:50,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:50,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752810968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:50,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:50,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:50,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:50,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:50,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752810970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:50,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:50,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:50,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:51,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752811076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:51,084 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:51,084 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:51,084 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:51,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:51,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752811086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:51,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752811086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:51,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:51,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:51,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:51,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:51,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:51,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:51,097 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/15ab4e4587db4fb98c3a969b11b42f3d as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/15ab4e4587db4fb98c3a969b11b42f3d 2024-11-16T10:25:51,100 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6c1e23082935a9ab78c6a0d7f447d65/B of b6c1e23082935a9ab78c6a0d7f447d65 into 15ab4e4587db4fb98c3a969b11b42f3d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:51,100 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:51,100 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., storeName=b6c1e23082935a9ab78c6a0d7f447d65/B, priority=12, startTime=1731752750676; duration=0sec 2024-11-16T10:25:51,101 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:51,101 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6c1e23082935a9ab78c6a0d7f447d65:B 2024-11-16T10:25:51,125 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/2b7f53029c3a4d8db4ce30a7bac4326a as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/2b7f53029c3a4d8db4ce30a7bac4326a 2024-11-16T10:25:51,128 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6c1e23082935a9ab78c6a0d7f447d65/C of b6c1e23082935a9ab78c6a0d7f447d65 into 2b7f53029c3a4d8db4ce30a7bac4326a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:51,128 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:51,128 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., storeName=b6c1e23082935a9ab78c6a0d7f447d65/C, priority=12, startTime=1731752750676; duration=0sec 2024-11-16T10:25:51,128 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:51,128 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6c1e23082935a9ab78c6a0d7f447d65:C 2024-11-16T10:25:51,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:51,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752811286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:51,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:51,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:51,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:51,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752811296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:51,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752811297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:51,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/46f50474e257447abb583af4e987dc0e 2024-11-16T10:25:51,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/8db3687c242946ad90c25bacba67b7ac is 50, key is test_row_0/B:col10/1731752750269/Put/seqid=0 2024-11-16T10:25:51,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742250_1426 (size=12151) 2024-11-16T10:25:51,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752811346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:51,352 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_2' in b6c1e23082935a9ab78c6a0d7f447d65 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:25:51,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:51,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:51,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T10:25:51,357 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:25:51,357 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:25:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-16T10:25:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T10:25:51,359 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:25:51,359 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:25:51,359 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:25:51,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752811356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:51,363 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_1' in b6c1e23082935a9ab78c6a0d7f447d65 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:25:51,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:51,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:51,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T10:25:51,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T10:25:51,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:51,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:51,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:51,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:51,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:51,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752811596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:51,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752811606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:51,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:51,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752811607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:51,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:51,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:51,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:51,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:51,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:51,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T10:25:51,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:51,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:51,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:51,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:51,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T10:25:51,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/8db3687c242946ad90c25bacba67b7ac 2024-11-16T10:25:51,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/481f3b9917aa4ea28e9ad107fe3b42c9 is 50, key is test_row_0/C:col10/1731752750269/Put/seqid=0 2024-11-16T10:25:51,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742251_1427 (size=12151) 2024-11-16T10:25:51,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T10:25:51,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:51,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:51,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:51,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:51,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:51,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T10:25:51,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:51,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:51,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:51,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:51,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:51,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:51,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T10:25:52,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:52,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752812107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:52,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:52,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:52,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:52,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:52,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752812116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:52,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:52,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T10:25:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:52,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752812116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:52,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:52,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:52,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:52,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:52,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:52,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:52,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/481f3b9917aa4ea28e9ad107fe3b42c9 2024-11-16T10:25:52,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/46f50474e257447abb583af4e987dc0e as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/46f50474e257447abb583af4e987dc0e 2024-11-16T10:25:52,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/46f50474e257447abb583af4e987dc0e, entries=200, sequenceid=170, filesize=14.2 K 2024-11-16T10:25:52,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/8db3687c242946ad90c25bacba67b7ac as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/8db3687c242946ad90c25bacba67b7ac 2024-11-16T10:25:52,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/8db3687c242946ad90c25bacba67b7ac, entries=150, sequenceid=170, filesize=11.9 K 2024-11-16T10:25:52,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/481f3b9917aa4ea28e9ad107fe3b42c9 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/481f3b9917aa4ea28e9ad107fe3b42c9 2024-11-16T10:25:52,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/481f3b9917aa4ea28e9ad107fe3b42c9, entries=150, sequenceid=170, filesize=11.9 K 2024-11-16T10:25:52,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for b6c1e23082935a9ab78c6a0d7f447d65 in 1254ms, sequenceid=170, compaction requested=false 2024-11-16T10:25:52,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:52,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T10:25:52,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:52,277 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing b6c1e23082935a9ab78c6a0d7f447d65 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T10:25:52,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/3f40159007cc4a119d168a44778aa8fd is 50, key is test_row_0/A:col10/1731752750957/Put/seqid=0 2024-11-16T10:25:52,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742252_1428 (size=12151) 2024-11-16T10:25:52,286 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/3f40159007cc4a119d168a44778aa8fd 2024-11-16T10:25:52,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/f24e67a60c6642aba03ed6f0ab1824fb is 50, key is test_row_0/B:col10/1731752750957/Put/seqid=0 2024-11-16T10:25:52,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742253_1429 (size=12151) 2024-11-16T10:25:52,303 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/f24e67a60c6642aba03ed6f0ab1824fb 2024-11-16T10:25:52,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/c3699d82e8d44cbdbe4a9e4770b39102 is 50, key is test_row_0/C:col10/1731752750957/Put/seqid=0 2024-11-16T10:25:52,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742254_1430 (size=12151) 2024-11-16T10:25:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T10:25:52,737 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/c3699d82e8d44cbdbe4a9e4770b39102 2024-11-16T10:25:52,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/3f40159007cc4a119d168a44778aa8fd as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/3f40159007cc4a119d168a44778aa8fd 2024-11-16T10:25:52,744 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/3f40159007cc4a119d168a44778aa8fd, entries=150, sequenceid=195, filesize=11.9 K 2024-11-16T10:25:52,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/f24e67a60c6642aba03ed6f0ab1824fb as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/f24e67a60c6642aba03ed6f0ab1824fb 2024-11-16T10:25:52,761 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/f24e67a60c6642aba03ed6f0ab1824fb, entries=150, sequenceid=195, filesize=11.9 K 2024-11-16T10:25:52,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/c3699d82e8d44cbdbe4a9e4770b39102 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/c3699d82e8d44cbdbe4a9e4770b39102 2024-11-16T10:25:52,774 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/c3699d82e8d44cbdbe4a9e4770b39102, entries=150, sequenceid=195, filesize=11.9 K 2024-11-16T10:25:52,774 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for b6c1e23082935a9ab78c6a0d7f447d65 in 497ms, sequenceid=195, compaction requested=true 2024-11-16T10:25:52,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:52,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:52,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-16T10:25:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-16T10:25:52,777 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-16T10:25:52,777 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4170 sec 2024-11-16T10:25:52,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.4210 sec 2024-11-16T10:25:53,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:53,148 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:25:53,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6c1e23082935a9ab78c6a0d7f447d65 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T10:25:53,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/972c9b733d0f4d9ca0a8898a72de20c3 is 50, key is test_row_0/A:col10/1731752753147/Put/seqid=0 2024-11-16T10:25:53,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742255_1431 (size=16931) 2024-11-16T10:25:53,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:53,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752813218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:53,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:53,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:53,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:53,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:53,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752813227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:53,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:53,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:53,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:53,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:53,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752813227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:53,241 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:53,241 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:53,241 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:53,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:53,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752813336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:53,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:53,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:53,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752813337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:53,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:53,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:53,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:53,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:53,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:53,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:53,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752813346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:53,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T10:25:53,506 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:25:53,507 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:25:53,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-16T10:25:53,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T10:25:53,509 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:25:53,509 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:25:53,509 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:25:53,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/972c9b733d0f4d9ca0a8898a72de20c3 2024-11-16T10:25:53,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:53,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752813557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:53,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:53,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752813557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:53,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:53,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:53,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752813557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:53,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:53,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:53,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/dd7fe84286e44fcca7bae25fe30a57c0 is 50, key is test_row_0/B:col10/1731752753147/Put/seqid=0 2024-11-16T10:25:53,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742256_1432 (size=12151) 2024-11-16T10:25:53,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T10:25:53,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T10:25:53,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:53,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:53,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:53,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:53,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:53,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:53,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T10:25:53,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:53,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:53,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:53,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:53,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:53,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:53,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T10:25:53,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:53,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752813876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:53,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:53,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752813876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:53,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:53,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752813876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:53,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:53,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:53,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:53,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:53,887 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:53,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:53,887 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:53,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:53,887 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:53,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T10:25:53,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:53,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:53,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:53,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:53,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:53,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/dd7fe84286e44fcca7bae25fe30a57c0 2024-11-16T10:25:53,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/4b98c830d5ab4d37a025c896e2044257 is 50, key is test_row_0/C:col10/1731752753147/Put/seqid=0 2024-11-16T10:25:53,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742257_1433 (size=12151) 2024-11-16T10:25:54,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T10:25:54,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:54,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:54,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:54,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:54,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:54,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T10:25:54,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T10:25:54,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:54,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:54,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:54,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:54,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:54,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/4b98c830d5ab4d37a025c896e2044257 2024-11-16T10:25:54,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/972c9b733d0f4d9ca0a8898a72de20c3 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/972c9b733d0f4d9ca0a8898a72de20c3 2024-11-16T10:25:54,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/972c9b733d0f4d9ca0a8898a72de20c3, entries=250, sequenceid=207, filesize=16.5 K 2024-11-16T10:25:54,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/dd7fe84286e44fcca7bae25fe30a57c0 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/dd7fe84286e44fcca7bae25fe30a57c0 2024-11-16T10:25:54,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:54,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752814396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:54,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:54,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:54,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:54,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:54,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752814396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:54,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:54,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752814396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:54,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:54,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:54,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:54,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:54,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:54,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:54,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/dd7fe84286e44fcca7bae25fe30a57c0, entries=150, sequenceid=207, filesize=11.9 K 2024-11-16T10:25:54,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/4b98c830d5ab4d37a025c896e2044257 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/4b98c830d5ab4d37a025c896e2044257 2024-11-16T10:25:54,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/4b98c830d5ab4d37a025c896e2044257, entries=150, sequenceid=207, filesize=11.9 K 2024-11-16T10:25:54,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for b6c1e23082935a9ab78c6a0d7f447d65 in 1261ms, sequenceid=207, compaction requested=true 2024-11-16T10:25:54,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:54,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6c1e23082935a9ab78c6a0d7f447d65:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:25:54,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:54,409 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:25:54,409 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:25:54,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6c1e23082935a9ab78c6a0d7f447d65:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:25:54,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:54,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6c1e23082935a9ab78c6a0d7f447d65:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:25:54,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:54,410 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48742 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:25:54,410 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:25:54,410 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): b6c1e23082935a9ab78c6a0d7f447d65/B is initiating minor compaction (all files) 2024-11-16T10:25:54,410 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): b6c1e23082935a9ab78c6a0d7f447d65/A is initiating minor compaction (all files) 2024-11-16T10:25:54,410 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6c1e23082935a9ab78c6a0d7f447d65/B in TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:54,410 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6c1e23082935a9ab78c6a0d7f447d65/A in TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:54,411 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/87ccfd6286ea46f98065ad24083f16f4, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/46f50474e257447abb583af4e987dc0e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/3f40159007cc4a119d168a44778aa8fd, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/972c9b733d0f4d9ca0a8898a72de20c3] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp, totalSize=54.6 K 2024-11-16T10:25:54,411 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/15ab4e4587db4fb98c3a969b11b42f3d, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/8db3687c242946ad90c25bacba67b7ac, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/f24e67a60c6642aba03ed6f0ab1824fb, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/dd7fe84286e44fcca7bae25fe30a57c0] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp, totalSize=47.6 K 2024-11-16T10:25:54,411 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 15ab4e4587db4fb98c3a969b11b42f3d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731752749084 2024-11-16T10:25:54,411 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87ccfd6286ea46f98065ad24083f16f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731752749084 2024-11-16T10:25:54,411 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 8db3687c242946ad90c25bacba67b7ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731752750242 2024-11-16T10:25:54,411 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46f50474e257447abb583af4e987dc0e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731752750242 2024-11-16T10:25:54,416 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting f24e67a60c6642aba03ed6f0ab1824fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731752750957 2024-11-16T10:25:54,416 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f40159007cc4a119d168a44778aa8fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731752750957 2024-11-16T10:25:54,417 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting dd7fe84286e44fcca7bae25fe30a57c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731752753140 2024-11-16T10:25:54,417 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 972c9b733d0f4d9ca0a8898a72de20c3, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731752753134 2024-11-16T10:25:54,423 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6c1e23082935a9ab78c6a0d7f447d65#A#compaction#368 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:54,423 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/fa491e5a1ab8477c8785e0315314fa5a is 50, key is test_row_0/A:col10/1731752753147/Put/seqid=0 2024-11-16T10:25:54,426 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6c1e23082935a9ab78c6a0d7f447d65#B#compaction#369 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:54,426 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/ebe990d451f3423a9297551cfad75004 is 50, key is test_row_0/B:col10/1731752753147/Put/seqid=0 2024-11-16T10:25:54,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T10:25:54,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:54,427 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing b6c1e23082935a9ab78c6a0d7f447d65 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T10:25:54,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742258_1434 (size=12425) 2024-11-16T10:25:54,441 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/fa491e5a1ab8477c8785e0315314fa5a as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/fa491e5a1ab8477c8785e0315314fa5a 2024-11-16T10:25:54,445 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6c1e23082935a9ab78c6a0d7f447d65/A of b6c1e23082935a9ab78c6a0d7f447d65 into fa491e5a1ab8477c8785e0315314fa5a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:54,445 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:54,445 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., storeName=b6c1e23082935a9ab78c6a0d7f447d65/A, priority=12, startTime=1731752754409; duration=0sec 2024-11-16T10:25:54,445 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:54,445 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6c1e23082935a9ab78c6a0d7f447d65:A 2024-11-16T10:25:54,445 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:25:54,446 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48742 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:25:54,446 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): b6c1e23082935a9ab78c6a0d7f447d65/C is initiating minor compaction (all files) 2024-11-16T10:25:54,446 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6c1e23082935a9ab78c6a0d7f447d65/C in TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:54,446 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/2b7f53029c3a4d8db4ce30a7bac4326a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/481f3b9917aa4ea28e9ad107fe3b42c9, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/c3699d82e8d44cbdbe4a9e4770b39102, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/4b98c830d5ab4d37a025c896e2044257] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp, totalSize=47.6 K 2024-11-16T10:25:54,446 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b7f53029c3a4d8db4ce30a7bac4326a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731752749084 2024-11-16T10:25:54,446 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 481f3b9917aa4ea28e9ad107fe3b42c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731752750242 2024-11-16T10:25:54,447 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3699d82e8d44cbdbe4a9e4770b39102, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731752750957 2024-11-16T10:25:54,447 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b98c830d5ab4d37a025c896e2044257, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731752753140 2024-11-16T10:25:54,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742259_1435 (size=12425) 2024-11-16T10:25:54,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/f5eeec26763b46658dc5dfc4d88b096e is 50, key is test_row_0/A:col10/1731752753225/Put/seqid=0 2024-11-16T10:25:54,478 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/ebe990d451f3423a9297551cfad75004 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/ebe990d451f3423a9297551cfad75004 2024-11-16T10:25:54,483 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6c1e23082935a9ab78c6a0d7f447d65/B of b6c1e23082935a9ab78c6a0d7f447d65 into ebe990d451f3423a9297551cfad75004(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:54,483 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:54,483 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., storeName=b6c1e23082935a9ab78c6a0d7f447d65/B, priority=12, startTime=1731752754409; duration=0sec 2024-11-16T10:25:54,483 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:54,483 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6c1e23082935a9ab78c6a0d7f447d65:B 2024-11-16T10:25:54,490 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6c1e23082935a9ab78c6a0d7f447d65#C#compaction#371 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:54,490 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/2083e59881464f4d9e06355f63593ab2 is 50, key is test_row_0/C:col10/1731752753147/Put/seqid=0 2024-11-16T10:25:54,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742260_1436 (size=12151) 2024-11-16T10:25:54,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742261_1437 (size=12425) 2024-11-16T10:25:54,505 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/2083e59881464f4d9e06355f63593ab2 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/2083e59881464f4d9e06355f63593ab2 2024-11-16T10:25:54,511 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6c1e23082935a9ab78c6a0d7f447d65/C of b6c1e23082935a9ab78c6a0d7f447d65 into 2083e59881464f4d9e06355f63593ab2(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:54,511 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:54,511 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., storeName=b6c1e23082935a9ab78c6a0d7f447d65/C, priority=12, startTime=1731752754410; duration=0sec 2024-11-16T10:25:54,512 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:54,512 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6c1e23082935a9ab78c6a0d7f447d65:C 2024-11-16T10:25:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T10:25:54,895 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/f5eeec26763b46658dc5dfc4d88b096e 2024-11-16T10:25:54,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/9d21225df6a844a7911f3ff15b942975 is 50, key is test_row_0/B:col10/1731752753225/Put/seqid=0 2024-11-16T10:25:54,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742262_1438 (size=12151) 2024-11-16T10:25:55,321 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/9d21225df6a844a7911f3ff15b942975 2024-11-16T10:25:55,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/2aea9100cc164a688e9052919b7111d6 is 50, key is test_row_0/C:col10/1731752753225/Put/seqid=0 2024-11-16T10:25:55,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742263_1439 (size=12151) 2024-11-16T10:25:55,373 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/2aea9100cc164a688e9052919b7111d6 2024-11-16T10:25:55,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/f5eeec26763b46658dc5dfc4d88b096e as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/f5eeec26763b46658dc5dfc4d88b096e 2024-11-16T10:25:55,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:55,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:55,387 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/f5eeec26763b46658dc5dfc4d88b096e, entries=150, sequenceid=231, filesize=11.9 K 2024-11-16T10:25:55,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/9d21225df6a844a7911f3ff15b942975 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/9d21225df6a844a7911f3ff15b942975 2024-11-16T10:25:55,394 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/9d21225df6a844a7911f3ff15b942975, entries=150, sequenceid=231, filesize=11.9 K 2024-11-16T10:25:55,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/2aea9100cc164a688e9052919b7111d6 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/2aea9100cc164a688e9052919b7111d6 2024-11-16T10:25:55,399 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/2aea9100cc164a688e9052919b7111d6, entries=150, sequenceid=231, filesize=11.9 K 2024-11-16T10:25:55,400 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=26.84 KB/27480 for b6c1e23082935a9ab78c6a0d7f447d65 in 972ms, sequenceid=231, compaction requested=false 2024-11-16T10:25:55,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:55,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:55,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-16T10:25:55,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-16T10:25:55,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-16T10:25:55,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8950 sec 2024-11-16T10:25:55,407 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.8990 sec 2024-11-16T10:25:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:55,412 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:25:55,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6c1e23082935a9ab78c6a0d7f447d65 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T10:25:55,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/ca243ad30a1542dda81038a3baf58b97 is 50, key is test_row_0/A:col10/1731752755408/Put/seqid=0 2024-11-16T10:25:55,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742264_1440 (size=14541) 2024-11-16T10:25:55,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/ca243ad30a1542dda81038a3baf58b97 2024-11-16T10:25:55,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/f157992dd90a4984b695529c990be5d7 is 50, key is test_row_0/B:col10/1731752755408/Put/seqid=0 2024-11-16T10:25:55,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742265_1441 (size=12151) 2024-11-16T10:25:55,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752815452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:55,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:55,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752815456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:55,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:55,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:55,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752815457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:55,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752815457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:55,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752815461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:55,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:55,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:55,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752815566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752815566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:55,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752815577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752815577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:55,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752815577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T10:25:55,669 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:25:55,670 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:25:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-16T10:25:55,672 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:25:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T10:25:55,673 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:25:55,673 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:25:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T10:25:55,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752815777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:55,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752815777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:55,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:55,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:55,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752815797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:55,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752815797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:55,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:55,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752815797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:55,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:55,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:55,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:55,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:55,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-16T10:25:55,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:55,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:55,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:55,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:55,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:55,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/f157992dd90a4984b695529c990be5d7 2024-11-16T10:25:55,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/6a15ec989f3a4944bbad7d99bebae30a is 50, key is test_row_0/C:col10/1731752755408/Put/seqid=0 2024-11-16T10:25:55,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742266_1442 (size=12151) 2024-11-16T10:25:55,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/6a15ec989f3a4944bbad7d99bebae30a 2024-11-16T10:25:55,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/ca243ad30a1542dda81038a3baf58b97 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/ca243ad30a1542dda81038a3baf58b97 2024-11-16T10:25:55,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/ca243ad30a1542dda81038a3baf58b97, entries=200, sequenceid=245, filesize=14.2 K 2024-11-16T10:25:55,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/f157992dd90a4984b695529c990be5d7 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/f157992dd90a4984b695529c990be5d7 2024-11-16T10:25:55,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/f157992dd90a4984b695529c990be5d7, entries=150, sequenceid=245, filesize=11.9 K 2024-11-16T10:25:55,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/6a15ec989f3a4944bbad7d99bebae30a as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/6a15ec989f3a4944bbad7d99bebae30a 2024-11-16T10:25:55,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/6a15ec989f3a4944bbad7d99bebae30a, entries=150, sequenceid=245, filesize=11.9 K 2024-11-16T10:25:55,910 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for b6c1e23082935a9ab78c6a0d7f447d65 in 498ms, sequenceid=245, compaction requested=true 2024-11-16T10:25:55,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:55,910 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:25:55,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6c1e23082935a9ab78c6a0d7f447d65:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:25:55,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:55,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6c1e23082935a9ab78c6a0d7f447d65:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:25:55,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:55,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6c1e23082935a9ab78c6a0d7f447d65:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:25:55,910 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:25:55,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:55,911 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39117 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:25:55,911 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): b6c1e23082935a9ab78c6a0d7f447d65/A is initiating minor compaction (all files) 2024-11-16T10:25:55,911 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6c1e23082935a9ab78c6a0d7f447d65/A in TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:55,911 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/fa491e5a1ab8477c8785e0315314fa5a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/f5eeec26763b46658dc5dfc4d88b096e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/ca243ad30a1542dda81038a3baf58b97] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp, totalSize=38.2 K 2024-11-16T10:25:55,912 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:25:55,912 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): b6c1e23082935a9ab78c6a0d7f447d65/B is initiating minor compaction (all files) 2024-11-16T10:25:55,912 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa491e5a1ab8477c8785e0315314fa5a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731752753140 2024-11-16T10:25:55,912 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6c1e23082935a9ab78c6a0d7f447d65/B in TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:55,912 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/ebe990d451f3423a9297551cfad75004, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/9d21225df6a844a7911f3ff15b942975, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/f157992dd90a4984b695529c990be5d7] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp, totalSize=35.9 K 2024-11-16T10:25:55,914 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting ebe990d451f3423a9297551cfad75004, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731752753140 2024-11-16T10:25:55,914 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5eeec26763b46658dc5dfc4d88b096e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731752753213 2024-11-16T10:25:55,915 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d21225df6a844a7911f3ff15b942975, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731752753213 2024-11-16T10:25:55,916 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca243ad30a1542dda81038a3baf58b97, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731752755380 2024-11-16T10:25:55,924 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting f157992dd90a4984b695529c990be5d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731752755403 2024-11-16T10:25:55,941 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6c1e23082935a9ab78c6a0d7f447d65#B#compaction#377 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:55,942 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/7f2a4cdf5ee4427eac25d5d61c42a2d0 is 50, key is test_row_0/B:col10/1731752755408/Put/seqid=0 2024-11-16T10:25:55,950 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6c1e23082935a9ab78c6a0d7f447d65#A#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:55,950 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/c8f75f7682e742b59593764c8422bd11 is 50, key is test_row_0/A:col10/1731752755408/Put/seqid=0 2024-11-16T10:25:55,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742267_1443 (size=12527) 2024-11-16T10:25:55,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-16T10:25:55,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:55,979 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing b6c1e23082935a9ab78c6a0d7f447d65 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T10:25:55,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742268_1444 (size=12527) 2024-11-16T10:25:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T10:25:55,994 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/7f2a4cdf5ee4427eac25d5d61c42a2d0 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/7f2a4cdf5ee4427eac25d5d61c42a2d0 2024-11-16T10:25:55,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/98a86ed6dabf4f748715ce4643d1376e is 50, key is test_row_0/A:col10/1731752755454/Put/seqid=0 2024-11-16T10:25:56,000 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/c8f75f7682e742b59593764c8422bd11 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/c8f75f7682e742b59593764c8422bd11 2024-11-16T10:25:56,003 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6c1e23082935a9ab78c6a0d7f447d65/B of b6c1e23082935a9ab78c6a0d7f447d65 into 7f2a4cdf5ee4427eac25d5d61c42a2d0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:56,003 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:56,003 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., storeName=b6c1e23082935a9ab78c6a0d7f447d65/B, priority=13, startTime=1731752755910; duration=0sec 2024-11-16T10:25:56,003 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:56,003 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6c1e23082935a9ab78c6a0d7f447d65:B 2024-11-16T10:25:56,003 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:25:56,005 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:25:56,005 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): b6c1e23082935a9ab78c6a0d7f447d65/C is initiating minor compaction (all files) 2024-11-16T10:25:56,005 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6c1e23082935a9ab78c6a0d7f447d65/C in TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:56,005 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/2083e59881464f4d9e06355f63593ab2, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/2aea9100cc164a688e9052919b7111d6, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/6a15ec989f3a4944bbad7d99bebae30a] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp, totalSize=35.9 K 2024-11-16T10:25:56,006 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 2083e59881464f4d9e06355f63593ab2, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731752753140 2024-11-16T10:25:56,007 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 2aea9100cc164a688e9052919b7111d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731752753213 2024-11-16T10:25:56,007 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6c1e23082935a9ab78c6a0d7f447d65/A of b6c1e23082935a9ab78c6a0d7f447d65 into c8f75f7682e742b59593764c8422bd11(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:56,007 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:56,007 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a15ec989f3a4944bbad7d99bebae30a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731752755403 2024-11-16T10:25:56,007 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., storeName=b6c1e23082935a9ab78c6a0d7f447d65/A, priority=13, startTime=1731752755910; duration=0sec 2024-11-16T10:25:56,007 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:56,007 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6c1e23082935a9ab78c6a0d7f447d65:A 2024-11-16T10:25:56,015 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6c1e23082935a9ab78c6a0d7f447d65#C#compaction#380 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:56,016 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/fcf2f30bb8c14e95ba86f9194c1cd861 is 50, key is test_row_0/C:col10/1731752755408/Put/seqid=0 2024-11-16T10:25:56,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742269_1445 (size=12301) 2024-11-16T10:25:56,019 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/98a86ed6dabf4f748715ce4643d1376e 2024-11-16T10:25:56,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/18dea4c60b8c47f085fce71c7ecc47f9 is 50, key is test_row_0/B:col10/1731752755454/Put/seqid=0 2024-11-16T10:25:56,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742270_1446 (size=12527) 2024-11-16T10:25:56,059 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/fcf2f30bb8c14e95ba86f9194c1cd861 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/fcf2f30bb8c14e95ba86f9194c1cd861 2024-11-16T10:25:56,064 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6c1e23082935a9ab78c6a0d7f447d65/C of b6c1e23082935a9ab78c6a0d7f447d65 into fcf2f30bb8c14e95ba86f9194c1cd861(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:56,064 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:56,065 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., storeName=b6c1e23082935a9ab78c6a0d7f447d65/C, priority=13, startTime=1731752755910; duration=0sec 2024-11-16T10:25:56,065 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:56,065 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6c1e23082935a9ab78c6a0d7f447d65:C 2024-11-16T10:25:56,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742271_1447 (size=12301) 2024-11-16T10:25:56,072 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/18dea4c60b8c47f085fce71c7ecc47f9 2024-11-16T10:25:56,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/03105004e5ac4d2faf31b253fdffcc69 is 50, key is test_row_0/C:col10/1731752755454/Put/seqid=0 2024-11-16T10:25:56,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742272_1448 (size=12301) 2024-11-16T10:25:56,087 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/03105004e5ac4d2faf31b253fdffcc69 2024-11-16T10:25:56,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:56,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:56,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/98a86ed6dabf4f748715ce4643d1376e as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/98a86ed6dabf4f748715ce4643d1376e 2024-11-16T10:25:56,097 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/98a86ed6dabf4f748715ce4643d1376e, entries=150, sequenceid=270, filesize=12.0 K 2024-11-16T10:25:56,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/18dea4c60b8c47f085fce71c7ecc47f9 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/18dea4c60b8c47f085fce71c7ecc47f9 2024-11-16T10:25:56,102 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/18dea4c60b8c47f085fce71c7ecc47f9, entries=150, sequenceid=270, filesize=12.0 K 2024-11-16T10:25:56,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/03105004e5ac4d2faf31b253fdffcc69 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/03105004e5ac4d2faf31b253fdffcc69 2024-11-16T10:25:56,110 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/03105004e5ac4d2faf31b253fdffcc69, entries=150, sequenceid=270, filesize=12.0 K 2024-11-16T10:25:56,111 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=33.54 KB/34350 for b6c1e23082935a9ab78c6a0d7f447d65 in 132ms, sequenceid=270, compaction requested=false 2024-11-16T10:25:56,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:56,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:56,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-16T10:25:56,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-16T10:25:56,115 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-16T10:25:56,115 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 440 msec 2024-11-16T10:25:56,117 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 446 msec 2024-11-16T10:25:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:56,119 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:25:56,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6c1e23082935a9ab78c6a0d7f447d65 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T10:25:56,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/4d8e9fb1be55429ea515ff8b59e7b6cc is 50, key is test_row_0/A:col10/1731752756112/Put/seqid=0 2024-11-16T10:25:56,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742273_1449 (size=14741) 2024-11-16T10:25:56,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/4d8e9fb1be55429ea515ff8b59e7b6cc 2024-11-16T10:25:56,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/aef5659e4fbf47ecb7d1500feb4740c5 is 50, key is test_row_0/B:col10/1731752756112/Put/seqid=0 2024-11-16T10:25:56,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742274_1450 (size=12301) 2024-11-16T10:25:56,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/aef5659e4fbf47ecb7d1500feb4740c5 2024-11-16T10:25:56,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/b5e738d93a8a46138aecff1d8252dfc2 is 50, key is test_row_0/C:col10/1731752756112/Put/seqid=0 2024-11-16T10:25:56,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:56,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752816171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:56,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:56,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:56,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752816172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:56,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742275_1451 (size=12301) 2024-11-16T10:25:56,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:56,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/b5e738d93a8a46138aecff1d8252dfc2 2024-11-16T10:25:56,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/4d8e9fb1be55429ea515ff8b59e7b6cc as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/4d8e9fb1be55429ea515ff8b59e7b6cc 2024-11-16T10:25:56,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:56,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752816184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:56,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:56,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:56,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752816193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:56,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:56,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752816194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:56,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/4d8e9fb1be55429ea515ff8b59e7b6cc, entries=200, sequenceid=284, filesize=14.4 K 2024-11-16T10:25:56,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/aef5659e4fbf47ecb7d1500feb4740c5 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/aef5659e4fbf47ecb7d1500feb4740c5 2024-11-16T10:25:56,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/aef5659e4fbf47ecb7d1500feb4740c5, entries=150, sequenceid=284, filesize=12.0 K 2024-11-16T10:25:56,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/b5e738d93a8a46138aecff1d8252dfc2 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/b5e738d93a8a46138aecff1d8252dfc2 2024-11-16T10:25:56,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/b5e738d93a8a46138aecff1d8252dfc2, entries=150, sequenceid=284, filesize=12.0 K 2024-11-16T10:25:56,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for b6c1e23082935a9ab78c6a0d7f447d65 in 103ms, sequenceid=284, compaction requested=true 2024-11-16T10:25:56,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:56,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6c1e23082935a9ab78c6a0d7f447d65:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:25:56,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:56,223 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:25:56,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6c1e23082935a9ab78c6a0d7f447d65:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:25:56,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:56,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6c1e23082935a9ab78c6a0d7f447d65:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:25:56,223 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:25:56,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:56,224 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39569 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:25:56,224 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): b6c1e23082935a9ab78c6a0d7f447d65/A is initiating minor compaction (all files) 2024-11-16T10:25:56,224 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6c1e23082935a9ab78c6a0d7f447d65/A in TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:56,224 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/c8f75f7682e742b59593764c8422bd11, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/98a86ed6dabf4f748715ce4643d1376e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/4d8e9fb1be55429ea515ff8b59e7b6cc] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp, totalSize=38.6 K 2024-11-16T10:25:56,224 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:25:56,224 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): b6c1e23082935a9ab78c6a0d7f447d65/B is initiating minor compaction (all files) 2024-11-16T10:25:56,224 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6c1e23082935a9ab78c6a0d7f447d65/B in TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:56,224 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/7f2a4cdf5ee4427eac25d5d61c42a2d0, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/18dea4c60b8c47f085fce71c7ecc47f9, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/aef5659e4fbf47ecb7d1500feb4740c5] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp, totalSize=36.3 K 2024-11-16T10:25:56,226 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f2a4cdf5ee4427eac25d5d61c42a2d0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731752755403 2024-11-16T10:25:56,226 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8f75f7682e742b59593764c8422bd11, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731752755403 2024-11-16T10:25:56,226 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98a86ed6dabf4f748715ce4643d1376e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731752755454 2024-11-16T10:25:56,226 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 18dea4c60b8c47f085fce71c7ecc47f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731752755454 2024-11-16T10:25:56,227 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d8e9fb1be55429ea515ff8b59e7b6cc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731752756104 2024-11-16T10:25:56,227 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting aef5659e4fbf47ecb7d1500feb4740c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731752756108 2024-11-16T10:25:56,234 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6c1e23082935a9ab78c6a0d7f447d65#B#compaction#386 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:56,235 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/32896a0631994f7fb75deaf122fac073 is 50, key is test_row_0/B:col10/1731752756112/Put/seqid=0 2024-11-16T10:25:56,237 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6c1e23082935a9ab78c6a0d7f447d65#A#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:56,238 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/ad03fd83403d46978b1122f33937916b is 50, key is test_row_0/A:col10/1731752756112/Put/seqid=0 2024-11-16T10:25:56,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742277_1453 (size=12779) 2024-11-16T10:25:56,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742276_1452 (size=12779) 2024-11-16T10:25:56,271 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/ad03fd83403d46978b1122f33937916b as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/ad03fd83403d46978b1122f33937916b 2024-11-16T10:25:56,275 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6c1e23082935a9ab78c6a0d7f447d65/A of b6c1e23082935a9ab78c6a0d7f447d65 into ad03fd83403d46978b1122f33937916b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:56,275 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:56,275 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., storeName=b6c1e23082935a9ab78c6a0d7f447d65/A, priority=13, startTime=1731752756223; duration=0sec 2024-11-16T10:25:56,275 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:56,275 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6c1e23082935a9ab78c6a0d7f447d65:A 2024-11-16T10:25:56,275 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:25:56,276 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:25:56,276 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): b6c1e23082935a9ab78c6a0d7f447d65/C is initiating minor compaction (all files) 2024-11-16T10:25:56,276 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6c1e23082935a9ab78c6a0d7f447d65/C in TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:56,276 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/fcf2f30bb8c14e95ba86f9194c1cd861, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/03105004e5ac4d2faf31b253fdffcc69, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/b5e738d93a8a46138aecff1d8252dfc2] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp, totalSize=36.3 K 2024-11-16T10:25:56,277 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcf2f30bb8c14e95ba86f9194c1cd861, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731752755403 2024-11-16T10:25:56,277 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03105004e5ac4d2faf31b253fdffcc69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731752755454 2024-11-16T10:25:56,277 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5e738d93a8a46138aecff1d8252dfc2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731752756108 2024-11-16T10:25:56,283 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6c1e23082935a9ab78c6a0d7f447d65#C#compaction#388 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:56,283 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/5309b62c3dfb4b49aebc33eefdc6dd1b is 50, key is test_row_0/C:col10/1731752756112/Put/seqid=0 2024-11-16T10:25:56,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742278_1454 (size=12779) 2024-11-16T10:25:56,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T10:25:56,297 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:25:56,297 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:25:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-16T10:25:56,299 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:25:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T10:25:56,299 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:25:56,299 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:25:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:56,307 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:25:56,307 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6c1e23082935a9ab78c6a0d7f447d65 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-16T10:25:56,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/e25aed6702ae41e9867c39a99170e47a is 50, key is test_row_0/A:col10/1731752756183/Put/seqid=0 2024-11-16T10:25:56,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742279_1455 (size=12301) 2024-11-16T10:25:56,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/e25aed6702ae41e9867c39a99170e47a 2024-11-16T10:25:56,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752816321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:56,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:56,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752816321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:56,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:56,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752816322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:56,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752816324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752816327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:56,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/267550b3d8404a3d95bc0338163e5f7f is 50, key is test_row_0/B:col10/1731752756183/Put/seqid=0 2024-11-16T10:25:56,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742280_1456 (size=12301) 2024-11-16T10:25:56,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/267550b3d8404a3d95bc0338163e5f7f 2024-11-16T10:25:56,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/e2feaeb60a374f60892dcf7ac7ae46f0 is 50, key is test_row_0/C:col10/1731752756183/Put/seqid=0 2024-11-16T10:25:56,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742281_1457 (size=12301) 2024-11-16T10:25:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T10:25:56,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T10:25:56,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:56,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752816447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:56,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:56,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752816447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:56,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:56,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752816447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:56,453 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:56,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:56,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752816447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:56,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:56,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752816447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:56,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:56,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:56,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:56,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:56,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:56,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:56,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:56,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:56,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:56,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:56,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T10:25:56,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:56,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:56,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:56,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:56,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:56,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T10:25:56,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752816656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:56,663 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,663 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,663 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:56,671 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/32896a0631994f7fb75deaf122fac073 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/32896a0631994f7fb75deaf122fac073 2024-11-16T10:25:56,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752816666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752816666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752816667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752816667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:56,676 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6c1e23082935a9ab78c6a0d7f447d65/B of b6c1e23082935a9ab78c6a0d7f447d65 into 32896a0631994f7fb75deaf122fac073(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:56,676 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:56,676 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., storeName=b6c1e23082935a9ab78c6a0d7f447d65/B, priority=13, startTime=1731752756223; duration=0sec 2024-11-16T10:25:56,676 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:56,676 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6c1e23082935a9ab78c6a0d7f447d65:B 2024-11-16T10:25:56,694 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/5309b62c3dfb4b49aebc33eefdc6dd1b as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/5309b62c3dfb4b49aebc33eefdc6dd1b 2024-11-16T10:25:56,697 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6c1e23082935a9ab78c6a0d7f447d65/C of b6c1e23082935a9ab78c6a0d7f447d65 into 5309b62c3dfb4b49aebc33eefdc6dd1b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:56,697 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:56,697 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., storeName=b6c1e23082935a9ab78c6a0d7f447d65/C, priority=13, startTime=1731752756223; duration=0sec 2024-11-16T10:25:56,697 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:56,697 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6c1e23082935a9ab78c6a0d7f447d65:C 2024-11-16T10:25:56,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T10:25:56,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:56,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:56,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:56,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:56,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:56,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:56,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/e2feaeb60a374f60892dcf7ac7ae46f0 2024-11-16T10:25:56,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/e25aed6702ae41e9867c39a99170e47a as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/e25aed6702ae41e9867c39a99170e47a 2024-11-16T10:25:56,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/e25aed6702ae41e9867c39a99170e47a, entries=150, sequenceid=313, filesize=12.0 K 2024-11-16T10:25:56,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/267550b3d8404a3d95bc0338163e5f7f as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/267550b3d8404a3d95bc0338163e5f7f 2024-11-16T10:25:56,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/267550b3d8404a3d95bc0338163e5f7f, entries=150, sequenceid=313, filesize=12.0 K 2024-11-16T10:25:56,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/e2feaeb60a374f60892dcf7ac7ae46f0 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/e2feaeb60a374f60892dcf7ac7ae46f0 2024-11-16T10:25:56,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/e2feaeb60a374f60892dcf7ac7ae46f0, entries=150, sequenceid=313, filesize=12.0 K 2024-11-16T10:25:56,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for b6c1e23082935a9ab78c6a0d7f447d65 in 521ms, sequenceid=313, compaction requested=false 2024-11-16T10:25:56,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:56,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T10:25:56,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:56,912 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing b6c1e23082935a9ab78c6a0d7f447d65 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-16T10:25:56,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/339537d3dc99465994d3a06700f741cb is 50, key is test_row_0/A:col10/1731752756320/Put/seqid=0 2024-11-16T10:25:56,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742282_1458 (size=9857) 2024-11-16T10:25:56,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T10:25:56,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:57,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752817016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:57,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:57,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752817017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:57,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:57,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752817022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:57,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:57,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:57,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:57,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752817025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:57,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:57,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:57,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752817026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:57,034 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,034 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,034 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:57,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752817136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:57,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752817136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752817136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:57,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752817136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:57,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752817136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:57,320 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/339537d3dc99465994d3a06700f741cb 2024-11-16T10:25:57,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/44c5966d002f43458c271176f71138f3 is 50, key is test_row_0/B:col10/1731752756320/Put/seqid=0 2024-11-16T10:25:57,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752817347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:57,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:57,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752817347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:57,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:57,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752817348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:57,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:57,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:57,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752817351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:57,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:57,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752817351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:57,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742283_1459 (size=9857) 2024-11-16T10:25:57,377 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/44c5966d002f43458c271176f71138f3 2024-11-16T10:25:57,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/db8262ca3b474e818aba6c82ca852042 is 50, key is test_row_0/C:col10/1731752756320/Put/seqid=0 2024-11-16T10:25:57,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742284_1460 (size=9857) 2024-11-16T10:25:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T10:25:57,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:57,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752817657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:57,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:57,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752817657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:57,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:57,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752817667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:57,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:57,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752817667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:57,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:57,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752817667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:57,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:57,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:57,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:57,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:57,806 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/db8262ca3b474e818aba6c82ca852042 2024-11-16T10:25:57,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/339537d3dc99465994d3a06700f741cb as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/339537d3dc99465994d3a06700f741cb 2024-11-16T10:25:57,814 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/339537d3dc99465994d3a06700f741cb, entries=100, sequenceid=323, filesize=9.6 K 2024-11-16T10:25:57,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/44c5966d002f43458c271176f71138f3 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/44c5966d002f43458c271176f71138f3 2024-11-16T10:25:57,823 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/44c5966d002f43458c271176f71138f3, entries=100, sequenceid=323, filesize=9.6 K 2024-11-16T10:25:57,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/db8262ca3b474e818aba6c82ca852042 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/db8262ca3b474e818aba6c82ca852042 2024-11-16T10:25:57,827 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/db8262ca3b474e818aba6c82ca852042, entries=100, sequenceid=323, filesize=9.6 K 2024-11-16T10:25:57,828 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for b6c1e23082935a9ab78c6a0d7f447d65 in 916ms, sequenceid=323, compaction requested=true 2024-11-16T10:25:57,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:57,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:57,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-16T10:25:57,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-16T10:25:57,833 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-16T10:25:57,833 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5310 sec 2024-11-16T10:25:57,835 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.5370 sec 2024-11-16T10:25:58,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:58,181 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:25:58,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6c1e23082935a9ab78c6a0d7f447d65 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-16T10:25:58,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/28bff8e5815149a6bd079446b0f716ec is 50, key is test_row_0/A:col10/1731752757023/Put/seqid=0 2024-11-16T10:25:58,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742285_1461 (size=14741) 2024-11-16T10:25:58,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:58,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:58,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752818187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:58,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752818187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:58,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:58,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752818195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:58,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:58,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752818196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:58,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:58,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:58,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:58,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:58,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752818197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:58,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:58,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:58,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:58,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:58,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752818306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:58,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:58,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:58,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:58,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752818306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:58,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:58,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:58,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:58,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:58,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752818316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:58,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:58,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:58,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:58,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T10:25:58,467 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:25:58,469 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:25:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-16T10:25:58,472 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:25:58,473 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:25:58,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:25:58,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T10:25:58,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:58,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752818517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:58,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:58,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:58,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:58,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752818517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:58,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:58,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:58,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:58,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:58,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:58,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752818526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:58,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:58,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:58,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:58,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T10:25:58,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/28bff8e5815149a6bd079446b0f716ec 2024-11-16T10:25:58,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/229bf036feb14a29b17d981fd946d593 is 50, key is test_row_0/B:col10/1731752757023/Put/seqid=0 2024-11-16T10:25:58,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742286_1462 (size=12301) 2024-11-16T10:25:58,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/229bf036feb14a29b17d981fd946d593 2024-11-16T10:25:58,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/ebecbe76d92541299ed5f248d189dcc3 is 50, key is test_row_0/C:col10/1731752757023/Put/seqid=0 2024-11-16T10:25:58,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T10:25:58,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:58,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:58,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:58,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:58,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:58,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:58,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742287_1463 (size=12301) 2024-11-16T10:25:58,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T10:25:58,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:58,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:58,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:58,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:58,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:58,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:58,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T10:25:58,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752818826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:58,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752818827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:58,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752818836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:58,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:58,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:58,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:58,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T10:25:58,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:58,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:58,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:58,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:58,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:58,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:25:59,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/ebecbe76d92541299ed5f248d189dcc3 2024-11-16T10:25:59,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/28bff8e5815149a6bd079446b0f716ec as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/28bff8e5815149a6bd079446b0f716ec 2024-11-16T10:25:59,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/28bff8e5815149a6bd079446b0f716ec, entries=200, sequenceid=352, filesize=14.4 K 2024-11-16T10:25:59,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/229bf036feb14a29b17d981fd946d593 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/229bf036feb14a29b17d981fd946d593 2024-11-16T10:25:59,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/229bf036feb14a29b17d981fd946d593, entries=150, sequenceid=352, filesize=12.0 K 2024-11-16T10:25:59,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/ebecbe76d92541299ed5f248d189dcc3 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/ebecbe76d92541299ed5f248d189dcc3 2024-11-16T10:25:59,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/ebecbe76d92541299ed5f248d189dcc3, entries=150, sequenceid=352, filesize=12.0 K 2024-11-16T10:25:59,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for b6c1e23082935a9ab78c6a0d7f447d65 in 894ms, sequenceid=352, compaction requested=true 2024-11-16T10:25:59,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:59,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6c1e23082935a9ab78c6a0d7f447d65:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:25:59,075 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:25:59,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:59,076 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:25:59,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6c1e23082935a9ab78c6a0d7f447d65:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:25:59,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:59,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6c1e23082935a9ab78c6a0d7f447d65:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:25:59,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:59,077 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49678 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:25:59,077 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): b6c1e23082935a9ab78c6a0d7f447d65/A is initiating minor compaction (all files) 2024-11-16T10:25:59,077 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47238 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:25:59,077 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): b6c1e23082935a9ab78c6a0d7f447d65/B is initiating minor compaction (all files) 2024-11-16T10:25:59,077 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6c1e23082935a9ab78c6a0d7f447d65/A in TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:59,077 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6c1e23082935a9ab78c6a0d7f447d65/B in TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:59,077 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/ad03fd83403d46978b1122f33937916b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/e25aed6702ae41e9867c39a99170e47a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/339537d3dc99465994d3a06700f741cb, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/28bff8e5815149a6bd079446b0f716ec] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp, totalSize=48.5 K 2024-11-16T10:25:59,077 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/32896a0631994f7fb75deaf122fac073, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/267550b3d8404a3d95bc0338163e5f7f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/44c5966d002f43458c271176f71138f3, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/229bf036feb14a29b17d981fd946d593] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp, totalSize=46.1 K 2024-11-16T10:25:59,078 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 32896a0631994f7fb75deaf122fac073, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731752756108 2024-11-16T10:25:59,078 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad03fd83403d46978b1122f33937916b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731752756108 2024-11-16T10:25:59,078 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 267550b3d8404a3d95bc0338163e5f7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731752756166 2024-11-16T10:25:59,078 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting e25aed6702ae41e9867c39a99170e47a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731752756166 2024-11-16T10:25:59,079 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 44c5966d002f43458c271176f71138f3, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731752756320 2024-11-16T10:25:59,079 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 339537d3dc99465994d3a06700f741cb, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731752756320 2024-11-16T10:25:59,079 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28bff8e5815149a6bd079446b0f716ec, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731752757016 2024-11-16T10:25:59,079 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 229bf036feb14a29b17d981fd946d593, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731752757016 2024-11-16T10:25:59,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T10:25:59,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:59,088 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing b6c1e23082935a9ab78c6a0d7f447d65 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-16T10:25:59,089 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6c1e23082935a9ab78c6a0d7f447d65#B#compaction#398 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:59,090 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/b6fe3f8e654244419ca6fa8dace58f40 is 50, key is test_row_0/B:col10/1731752757023/Put/seqid=0 2024-11-16T10:25:59,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T10:25:59,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/44bb40ed60c74a8fbce7385448940b4b is 50, key is test_row_0/A:col10/1731752758195/Put/seqid=0 2024-11-16T10:25:59,098 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6c1e23082935a9ab78c6a0d7f447d65#A#compaction#399 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:59,098 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/5175b4c863f5450eb4bb5c4572bfc78b is 50, key is test_row_0/A:col10/1731752757023/Put/seqid=0 2024-11-16T10:25:59,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742288_1464 (size=12915) 2024-11-16T10:25:59,106 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/b6fe3f8e654244419ca6fa8dace58f40 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/b6fe3f8e654244419ca6fa8dace58f40 2024-11-16T10:25:59,109 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6c1e23082935a9ab78c6a0d7f447d65/B of b6c1e23082935a9ab78c6a0d7f447d65 into b6fe3f8e654244419ca6fa8dace58f40(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:59,109 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:59,109 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., storeName=b6c1e23082935a9ab78c6a0d7f447d65/B, priority=12, startTime=1731752759075; duration=0sec 2024-11-16T10:25:59,109 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:25:59,109 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6c1e23082935a9ab78c6a0d7f447d65:B 2024-11-16T10:25:59,109 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:25:59,110 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47238 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:25:59,110 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): b6c1e23082935a9ab78c6a0d7f447d65/C is initiating minor compaction (all files) 2024-11-16T10:25:59,110 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6c1e23082935a9ab78c6a0d7f447d65/C in TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:25:59,110 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/5309b62c3dfb4b49aebc33eefdc6dd1b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/e2feaeb60a374f60892dcf7ac7ae46f0, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/db8262ca3b474e818aba6c82ca852042, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/ebecbe76d92541299ed5f248d189dcc3] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp, totalSize=46.1 K 2024-11-16T10:25:59,111 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 5309b62c3dfb4b49aebc33eefdc6dd1b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731752756108 2024-11-16T10:25:59,111 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting e2feaeb60a374f60892dcf7ac7ae46f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731752756166 2024-11-16T10:25:59,111 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting db8262ca3b474e818aba6c82ca852042, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731752756320 2024-11-16T10:25:59,111 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting ebecbe76d92541299ed5f248d189dcc3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731752757016 2024-11-16T10:25:59,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742289_1465 (size=9857) 2024-11-16T10:25:59,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742290_1466 (size=12915) 2024-11-16T10:25:59,133 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6c1e23082935a9ab78c6a0d7f447d65#C#compaction#401 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:25:59,134 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/0ad753e11b7744c18d7f02318a658cae is 50, key is test_row_0/C:col10/1731752757023/Put/seqid=0 2024-11-16T10:25:59,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742291_1467 (size=12915) 2024-11-16T10:25:59,161 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/0ad753e11b7744c18d7f02318a658cae as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/0ad753e11b7744c18d7f02318a658cae 2024-11-16T10:25:59,164 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6c1e23082935a9ab78c6a0d7f447d65/C of b6c1e23082935a9ab78c6a0d7f447d65 into 0ad753e11b7744c18d7f02318a658cae(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:59,164 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:59,164 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., storeName=b6c1e23082935a9ab78c6a0d7f447d65/C, priority=12, startTime=1731752759076; duration=0sec 2024-11-16T10:25:59,165 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:59,165 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6c1e23082935a9ab78c6a0d7f447d65:C 2024-11-16T10:25:59,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:25:59,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:25:59,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:59,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752819293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:59,299 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:59,299 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:59,299 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:59,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:59,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752819296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:59,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:59,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:59,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:59,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:59,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752819346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:59,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:59,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:59,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:59,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752819346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:59,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:59,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:59,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:59,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:59,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752819356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:59,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:59,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:59,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:59,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:59,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752819406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:59,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:59,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:59,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:59,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:59,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752819406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:59,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:59,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:59,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:59,518 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/44bb40ed60c74a8fbce7385448940b4b 2024-11-16T10:25:59,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/c4441e36bac44e8694209ff81c8f57c3 is 50, key is test_row_0/B:col10/1731752758195/Put/seqid=0 2024-11-16T10:25:59,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742292_1468 (size=9857) 2024-11-16T10:25:59,532 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/5175b4c863f5450eb4bb5c4572bfc78b as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/5175b4c863f5450eb4bb5c4572bfc78b 2024-11-16T10:25:59,535 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6c1e23082935a9ab78c6a0d7f447d65/A of b6c1e23082935a9ab78c6a0d7f447d65 into 5175b4c863f5450eb4bb5c4572bfc78b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:25:59,535 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:25:59,535 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., storeName=b6c1e23082935a9ab78c6a0d7f447d65/A, priority=12, startTime=1731752759075; duration=0sec 2024-11-16T10:25:59,535 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:25:59,535 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6c1e23082935a9ab78c6a0d7f447d65:A 2024-11-16T10:25:59,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T10:25:59,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:59,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752819616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:59,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:59,620 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:59,620 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:59,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752819616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:59,620 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:59,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:59,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:59,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:59,926 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/c4441e36bac44e8694209ff81c8f57c3 2024-11-16T10:25:59,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:59,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752819926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:59,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:25:59,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752819926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:25:59,928 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:59,928 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:59,928 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:59,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:59,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:25:59,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:25:59,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/95bb7cde876e4522b38fe065ec44a431 is 50, key is test_row_0/C:col10/1731752758195/Put/seqid=0 2024-11-16T10:25:59,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742293_1469 (size=9857) 2024-11-16T10:26:00,335 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/95bb7cde876e4522b38fe065ec44a431 2024-11-16T10:26:00,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/44bb40ed60c74a8fbce7385448940b4b as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/44bb40ed60c74a8fbce7385448940b4b 2024-11-16T10:26:00,343 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/44bb40ed60c74a8fbce7385448940b4b, entries=100, sequenceid=359, filesize=9.6 K 2024-11-16T10:26:00,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/c4441e36bac44e8694209ff81c8f57c3 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/c4441e36bac44e8694209ff81c8f57c3 2024-11-16T10:26:00,346 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/c4441e36bac44e8694209ff81c8f57c3, entries=100, sequenceid=359, filesize=9.6 K 2024-11-16T10:26:00,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/95bb7cde876e4522b38fe065ec44a431 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/95bb7cde876e4522b38fe065ec44a431 2024-11-16T10:26:00,350 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/95bb7cde876e4522b38fe065ec44a431, entries=100, sequenceid=359, filesize=9.6 K 2024-11-16T10:26:00,351 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=174.43 KB/178620 for b6c1e23082935a9ab78c6a0d7f447d65 in 1263ms, sequenceid=359, compaction requested=false 2024-11-16T10:26:00,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:26:00,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:00,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-16T10:26:00,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-16T10:26:00,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-16T10:26:00,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8800 sec 2024-11-16T10:26:00,355 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.8850 sec 2024-11-16T10:26:00,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:26:00,372 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:26:00,372 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6c1e23082935a9ab78c6a0d7f447d65 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-16T10:26:00,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/039e2bf1d51040f2a76cb080a2e59938 is 50, key is test_row_0/A:col10/1731752759291/Put/seqid=0 2024-11-16T10:26:00,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742294_1470 (size=14741) 2024-11-16T10:26:00,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752820376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:00,382 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_0' in b6c1e23082935a9ab78c6a0d7f447d65 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:26:00,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:00,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:00,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:00,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752820381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:00,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:00,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:00,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:00,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:00,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752820381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:00,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:00,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:00,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:00,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:00,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752820436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:00,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:00,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:00,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:00,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752820436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:00,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:00,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:00,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:00,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:00,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:00,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752820486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:00,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:00,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:00,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:00,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:00,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752820496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:00,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:00,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:00,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T10:26:00,617 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:26:00,617 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:26:00,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-16T10:26:00,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T10:26:00,619 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:26:00,619 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:26:00,619 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:26:00,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752820697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:00,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:00,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:00,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:00,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752820706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:00,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:00,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:00,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:00,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T10:26:00,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T10:26:00,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:00,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:26:00,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:00,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:00,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:00,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:00,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/039e2bf1d51040f2a76cb080a2e59938 2024-11-16T10:26:00,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/586e94aa73b94b9f9b14a909ee558ccc is 50, key is test_row_0/B:col10/1731752759291/Put/seqid=0 2024-11-16T10:26:00,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742295_1471 (size=12301) 2024-11-16T10:26:00,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/586e94aa73b94b9f9b14a909ee558ccc 2024-11-16T10:26:00,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/aba3eb2464024701996494c451beb75a is 50, key is test_row_0/C:col10/1731752759291/Put/seqid=0 2024-11-16T10:26:00,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742296_1472 (size=12301) 2024-11-16T10:26:00,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T10:26:00,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:00,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:26:00,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:00,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:00,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:00,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:00,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T10:26:01,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752821006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:01,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:01,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752821019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:01,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:01,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:01,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:01,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T10:26:01,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:01,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:26:01,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:01,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:01,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:01,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:01,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/aba3eb2464024701996494c451beb75a 2024-11-16T10:26:01,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/039e2bf1d51040f2a76cb080a2e59938 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/039e2bf1d51040f2a76cb080a2e59938 2024-11-16T10:26:01,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/039e2bf1d51040f2a76cb080a2e59938, entries=200, sequenceid=392, filesize=14.4 K 2024-11-16T10:26:01,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/586e94aa73b94b9f9b14a909ee558ccc as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/586e94aa73b94b9f9b14a909ee558ccc 2024-11-16T10:26:01,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/586e94aa73b94b9f9b14a909ee558ccc, entries=150, sequenceid=392, filesize=12.0 K 2024-11-16T10:26:01,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/aba3eb2464024701996494c451beb75a as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/aba3eb2464024701996494c451beb75a 2024-11-16T10:26:01,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/aba3eb2464024701996494c451beb75a, entries=150, sequenceid=392, filesize=12.0 K 2024-11-16T10:26:01,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for b6c1e23082935a9ab78c6a0d7f447d65 in 850ms, sequenceid=392, compaction requested=true 2024-11-16T10:26:01,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:26:01,223 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:26:01,223 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37513 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:26:01,223 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): b6c1e23082935a9ab78c6a0d7f447d65/A is initiating minor compaction (all files) 2024-11-16T10:26:01,224 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6c1e23082935a9ab78c6a0d7f447d65/A in TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:01,224 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/5175b4c863f5450eb4bb5c4572bfc78b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/44bb40ed60c74a8fbce7385448940b4b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/039e2bf1d51040f2a76cb080a2e59938] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp, totalSize=36.6 K 2024-11-16T10:26:01,224 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5175b4c863f5450eb4bb5c4572bfc78b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731752757016 2024-11-16T10:26:01,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6c1e23082935a9ab78c6a0d7f447d65:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:26:01,224 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44bb40ed60c74a8fbce7385448940b4b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731752758182 2024-11-16T10:26:01,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:01,224 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:26:01,225 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 039e2bf1d51040f2a76cb080a2e59938, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731752759273 2024-11-16T10:26:01,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6c1e23082935a9ab78c6a0d7f447d65:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:26:01,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:01,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6c1e23082935a9ab78c6a0d7f447d65:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:26:01,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:26:01,226 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:26:01,226 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): b6c1e23082935a9ab78c6a0d7f447d65/B is initiating minor compaction (all files) 2024-11-16T10:26:01,226 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6c1e23082935a9ab78c6a0d7f447d65/B in TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:01,226 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/b6fe3f8e654244419ca6fa8dace58f40, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/c4441e36bac44e8694209ff81c8f57c3, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/586e94aa73b94b9f9b14a909ee558ccc] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp, totalSize=34.3 K 2024-11-16T10:26:01,226 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting b6fe3f8e654244419ca6fa8dace58f40, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731752757016 2024-11-16T10:26:01,227 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting c4441e36bac44e8694209ff81c8f57c3, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731752758182 2024-11-16T10:26:01,227 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 586e94aa73b94b9f9b14a909ee558ccc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731752759291 2024-11-16T10:26:01,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T10:26:01,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:01,230 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing b6c1e23082935a9ab78c6a0d7f447d65 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-16T10:26:01,232 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6c1e23082935a9ab78c6a0d7f447d65#A#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:26:01,232 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/58b3d40c34e44800a8b6aae0bcb3df4b is 50, key is test_row_0/A:col10/1731752759291/Put/seqid=0 2024-11-16T10:26:01,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/a3aa4eb5ca2946528b14516333c88dd4 is 50, key is test_row_1/A:col10/1731752760380/Put/seqid=0 2024-11-16T10:26:01,243 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6c1e23082935a9ab78c6a0d7f447d65#B#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:26:01,244 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/8e06e1a4f319450581116278380dba83 is 50, key is test_row_0/B:col10/1731752759291/Put/seqid=0 2024-11-16T10:26:01,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T10:26:01,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742297_1473 (size=13017) 2024-11-16T10:26:01,255 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/58b3d40c34e44800a8b6aae0bcb3df4b as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/58b3d40c34e44800a8b6aae0bcb3df4b 2024-11-16T10:26:01,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742298_1474 (size=7415) 2024-11-16T10:26:01,261 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/a3aa4eb5ca2946528b14516333c88dd4 2024-11-16T10:26:01,263 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6c1e23082935a9ab78c6a0d7f447d65/A of b6c1e23082935a9ab78c6a0d7f447d65 into 58b3d40c34e44800a8b6aae0bcb3df4b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:26:01,263 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:26:01,263 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., storeName=b6c1e23082935a9ab78c6a0d7f447d65/A, priority=13, startTime=1731752761222; duration=0sec 2024-11-16T10:26:01,264 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:26:01,264 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6c1e23082935a9ab78c6a0d7f447d65:A 2024-11-16T10:26:01,264 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:26:01,265 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:26:01,265 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): b6c1e23082935a9ab78c6a0d7f447d65/C is initiating minor compaction (all files) 2024-11-16T10:26:01,265 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6c1e23082935a9ab78c6a0d7f447d65/C in TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:01,266 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/0ad753e11b7744c18d7f02318a658cae, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/95bb7cde876e4522b38fe065ec44a431, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/aba3eb2464024701996494c451beb75a] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp, totalSize=34.3 K 2024-11-16T10:26:01,266 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ad753e11b7744c18d7f02318a658cae, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731752757016 2024-11-16T10:26:01,266 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95bb7cde876e4522b38fe065ec44a431, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731752758182 2024-11-16T10:26:01,266 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting aba3eb2464024701996494c451beb75a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731752759291 2024-11-16T10:26:01,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/9528ff32c7d743b086f59f8a10f14a3b is 50, key is test_row_1/B:col10/1731752760380/Put/seqid=0 2024-11-16T10:26:01,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742299_1475 (size=13017) 2024-11-16T10:26:01,290 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6c1e23082935a9ab78c6a0d7f447d65#C#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:26:01,290 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/c73f352bc0d14621b2d0efe2415a7a7e is 50, key is test_row_0/C:col10/1731752759291/Put/seqid=0 2024-11-16T10:26:01,294 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/8e06e1a4f319450581116278380dba83 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/8e06e1a4f319450581116278380dba83 2024-11-16T10:26:01,298 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6c1e23082935a9ab78c6a0d7f447d65/B of b6c1e23082935a9ab78c6a0d7f447d65 into 8e06e1a4f319450581116278380dba83(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:26:01,298 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:26:01,298 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., storeName=b6c1e23082935a9ab78c6a0d7f447d65/B, priority=13, startTime=1731752761224; duration=0sec 2024-11-16T10:26:01,298 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:01,298 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6c1e23082935a9ab78c6a0d7f447d65:B 2024-11-16T10:26:01,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742300_1476 (size=7415) 2024-11-16T10:26:01,331 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/9528ff32c7d743b086f59f8a10f14a3b 2024-11-16T10:26:01,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742301_1477 (size=13017) 2024-11-16T10:26:01,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/c4a0e9f773394a908db5395b2094123b is 50, key is test_row_1/C:col10/1731752760380/Put/seqid=0 2024-11-16T10:26:01,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742302_1478 (size=7415) 2024-11-16T10:26:01,382 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/c4a0e9f773394a908db5395b2094123b 2024-11-16T10:26:01,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/a3aa4eb5ca2946528b14516333c88dd4 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/a3aa4eb5ca2946528b14516333c88dd4 2024-11-16T10:26:01,396 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/a3aa4eb5ca2946528b14516333c88dd4, entries=50, sequenceid=398, filesize=7.2 K 2024-11-16T10:26:01,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/9528ff32c7d743b086f59f8a10f14a3b as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/9528ff32c7d743b086f59f8a10f14a3b 2024-11-16T10:26:01,401 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/9528ff32c7d743b086f59f8a10f14a3b, entries=50, sequenceid=398, filesize=7.2 K 2024-11-16T10:26:01,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/c4a0e9f773394a908db5395b2094123b as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/c4a0e9f773394a908db5395b2094123b 2024-11-16T10:26:01,405 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/c4a0e9f773394a908db5395b2094123b, entries=50, sequenceid=398, filesize=7.2 K 2024-11-16T10:26:01,406 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for b6c1e23082935a9ab78c6a0d7f447d65 in 176ms, sequenceid=398, compaction requested=false 2024-11-16T10:26:01,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:26:01,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:01,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-16T10:26:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-16T10:26:01,409 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-16T10:26:01,409 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 788 msec 2024-11-16T10:26:01,411 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 793 msec 2024-11-16T10:26:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:26:01,469 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:26:01,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6c1e23082935a9ab78c6a0d7f447d65 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T10:26:01,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/00bcde5576a649f0aba9d063f3a5f9f3 is 50, key is test_row_0/A:col10/1731752761468/Put/seqid=0 2024-11-16T10:26:01,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742303_1479 (size=14741) 2024-11-16T10:26:01,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:01,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752821550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:01,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:01,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:01,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752821551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:01,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:01,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:01,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:01,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:01,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752821561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:01,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752821561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:01,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752821666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:01,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:01,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752821666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:01,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:01,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:01,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:01,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:01,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:01,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752821666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:01,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:01,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:01,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:01,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752821666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:01,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:01,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:01,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:01,743 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/c73f352bc0d14621b2d0efe2415a7a7e as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/c73f352bc0d14621b2d0efe2415a7a7e 2024-11-16T10:26:01,747 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6c1e23082935a9ab78c6a0d7f447d65/C of b6c1e23082935a9ab78c6a0d7f447d65 into c73f352bc0d14621b2d0efe2415a7a7e(size=12.7 K), total size for store is 20.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:26:01,748 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:26:01,748 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., storeName=b6c1e23082935a9ab78c6a0d7f447d65/C, priority=13, startTime=1731752761225; duration=0sec 2024-11-16T10:26:01,748 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:01,748 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6c1e23082935a9ab78c6a0d7f447d65:C 2024-11-16T10:26:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T10:26:01,756 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:26:01,756 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:26:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-16T10:26:01,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T10:26:01,758 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:26:01,758 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:26:01,758 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:26:01,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T10:26:01,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:01,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752821876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:01,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:01,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:01,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752821876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:01,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:01,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:01,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:01,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:01,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:01,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/00bcde5576a649f0aba9d063f3a5f9f3 2024-11-16T10:26:01,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/e5ab4f6948e44d499094925b65654baa is 50, key is test_row_0/B:col10/1731752761468/Put/seqid=0 2024-11-16T10:26:01,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:01,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742304_1480 (size=12301) 2024-11-16T10:26:01,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752821878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:01,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752821886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:01,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:01,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:01,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:01,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:01,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:01,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:01,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T10:26:01,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:01,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:26:01,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:01,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:01,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:02,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T10:26:02,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:02,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:26:02,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:02,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:02,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:02,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T10:26:02,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:02,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752822187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:02,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:02,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:02,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:02,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752822187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:02,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:02,194 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:02,194 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:02,194 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:02,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:02,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752822196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:02,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:02,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752822196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:02,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:02,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:02,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:02,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:02,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:02,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:02,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T10:26:02,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:02,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:26:02,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:02,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:02,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:02,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:02,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/e5ab4f6948e44d499094925b65654baa 2024-11-16T10:26:02,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/cf4c530d87eb45aeb6c7c60a875e41e0 is 50, key is test_row_0/C:col10/1731752761468/Put/seqid=0 2024-11-16T10:26:02,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742305_1481 (size=12301) 2024-11-16T10:26:02,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T10:26:02,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:02,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:26:02,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:02,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:02,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:02,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:02,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T10:26:02,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38866 deadline: 1731752822396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:02,400 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_0' in b6c1e23082935a9ab78c6a0d7f447d65 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:26:02,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:02,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:02,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:02,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T10:26:02,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:02,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:26:02,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:02,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:02,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:02,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:02,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T10:26:02,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:02,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:26:02,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:02,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:02,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:02,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:02,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/cf4c530d87eb45aeb6c7c60a875e41e0 2024-11-16T10:26:02,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/00bcde5576a649f0aba9d063f3a5f9f3 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/00bcde5576a649f0aba9d063f3a5f9f3 2024-11-16T10:26:02,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/00bcde5576a649f0aba9d063f3a5f9f3, entries=200, sequenceid=411, filesize=14.4 K 2024-11-16T10:26:02,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/e5ab4f6948e44d499094925b65654baa as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/e5ab4f6948e44d499094925b65654baa 2024-11-16T10:26:02,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/e5ab4f6948e44d499094925b65654baa, entries=150, sequenceid=411, filesize=12.0 K 2024-11-16T10:26:02,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/cf4c530d87eb45aeb6c7c60a875e41e0 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/cf4c530d87eb45aeb6c7c60a875e41e0 2024-11-16T10:26:02,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/cf4c530d87eb45aeb6c7c60a875e41e0, entries=150, sequenceid=411, filesize=12.0 K 2024-11-16T10:26:02,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:02,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752822706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:02,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:02,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752822706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:02,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:02,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:02,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:02,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:02,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:02,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:02,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for b6c1e23082935a9ab78c6a0d7f447d65 in 1246ms, sequenceid=411, compaction requested=true 2024-11-16T10:26:02,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:26:02,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6c1e23082935a9ab78c6a0d7f447d65:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:26:02,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:02,715 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:26:02,715 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:26:02,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6c1e23082935a9ab78c6a0d7f447d65:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:26:02,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:02,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6c1e23082935a9ab78c6a0d7f447d65:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:26:02,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:26:02,716 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35173 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:26:02,716 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32733 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:26:02,716 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): b6c1e23082935a9ab78c6a0d7f447d65/A is initiating minor compaction (all files) 2024-11-16T10:26:02,716 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): b6c1e23082935a9ab78c6a0d7f447d65/B is initiating minor compaction (all files) 2024-11-16T10:26:02,717 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6c1e23082935a9ab78c6a0d7f447d65/A in TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:02,717 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6c1e23082935a9ab78c6a0d7f447d65/B in TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:02,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:26:02,717 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/8e06e1a4f319450581116278380dba83, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/9528ff32c7d743b086f59f8a10f14a3b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/e5ab4f6948e44d499094925b65654baa] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp, totalSize=32.0 K 2024-11-16T10:26:02,717 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/58b3d40c34e44800a8b6aae0bcb3df4b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/a3aa4eb5ca2946528b14516333c88dd4, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/00bcde5576a649f0aba9d063f3a5f9f3] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp, totalSize=34.3 K 2024-11-16T10:26:02,717 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:26:02,717 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6c1e23082935a9ab78c6a0d7f447d65 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-16T10:26:02,717 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58b3d40c34e44800a8b6aae0bcb3df4b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731752759291 2024-11-16T10:26:02,717 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e06e1a4f319450581116278380dba83, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731752759291 2024-11-16T10:26:02,717 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3aa4eb5ca2946528b14516333c88dd4, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731752760380 2024-11-16T10:26:02,717 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 9528ff32c7d743b086f59f8a10f14a3b, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731752760380 2024-11-16T10:26:02,717 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00bcde5576a649f0aba9d063f3a5f9f3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731752761454 2024-11-16T10:26:02,717 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting e5ab4f6948e44d499094925b65654baa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731752761454 2024-11-16T10:26:02,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/b5739ddb7cbe472bac406efcba40d2b2 is 50, key is test_row_0/A:col10/1731752762716/Put/seqid=0 2024-11-16T10:26:02,733 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6c1e23082935a9ab78c6a0d7f447d65#A#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:26:02,734 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6c1e23082935a9ab78c6a0d7f447d65#B#compaction#418 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:26:02,734 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/d83570b5acda413ea1ad0b1a4ff89b4b is 50, key is test_row_0/A:col10/1731752761468/Put/seqid=0 2024-11-16T10:26:02,734 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/d1c623593e934e059810f8903e1c42ab is 50, key is test_row_0/B:col10/1731752761468/Put/seqid=0 2024-11-16T10:26:02,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742307_1483 (size=13119) 2024-11-16T10:26:02,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742308_1484 (size=13119) 2024-11-16T10:26:02,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742306_1482 (size=17181) 2024-11-16T10:26:02,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:02,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752822740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:02,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:02,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752822745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:02,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:02,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:02,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:02,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:02,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:02,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:02,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T10:26:02,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:02,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:26:02,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:02,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:02,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:02,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:02,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:02,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752822857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:02,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:02,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:02,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:02,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:02,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752822857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:02,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:02,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:02,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:02,896 INFO [Thread-1799 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1799. 2024-11-16T10:26:02,896 DEBUG [Thread-1799 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:26:02,896 DEBUG [Thread-1799 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:02,897 DEBUG [Thread-1799 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:02,897 INFO [Thread-1801 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1801. 2024-11-16T10:26:02,897 DEBUG [Thread-1801 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:26:02,897 DEBUG [Thread-1801 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:02,897 DEBUG [Thread-1801 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:02,897 INFO [Thread-1797 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1797. 2024-11-16T10:26:02,897 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:26:02,897 DEBUG [Thread-1797 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:26:02,897 DEBUG [Thread-1797 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:02,897 DEBUG [Thread-1797 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:02,897 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:26:02,897 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:26:02,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T10:26:02,909 INFO [Thread-1793 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1793. 2024-11-16T10:26:02,909 INFO [Thread-1795 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1795. 2024-11-16T10:26:02,909 DEBUG [Thread-1793 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:26:02,909 DEBUG [Thread-1795 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:26:02,909 DEBUG [Thread-1795 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:02,909 DEBUG [Thread-1793 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:02,909 DEBUG [Thread-1793 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:02,909 DEBUG [Thread-1795 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:02,909 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:26:02,909 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:26:02,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T10:26:02,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:02,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:26:02,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:02,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:02,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:02,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:03,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:03,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752823067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:03,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:03,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:03,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:03,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:03,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752823076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:03,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:03,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:03,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:03,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T10:26:03,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:03,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:26:03,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:03,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:03,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:03,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:03,143 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/d83570b5acda413ea1ad0b1a4ff89b4b as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/d83570b5acda413ea1ad0b1a4ff89b4b 2024-11-16T10:26:03,145 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/d1c623593e934e059810f8903e1c42ab as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/d1c623593e934e059810f8903e1c42ab 2024-11-16T10:26:03,147 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6c1e23082935a9ab78c6a0d7f447d65/A of b6c1e23082935a9ab78c6a0d7f447d65 into d83570b5acda413ea1ad0b1a4ff89b4b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:26:03,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/b5739ddb7cbe472bac406efcba40d2b2 2024-11-16T10:26:03,147 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:26:03,147 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., storeName=b6c1e23082935a9ab78c6a0d7f447d65/A, priority=13, startTime=1731752762715; duration=0sec 2024-11-16T10:26:03,147 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:26:03,147 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6c1e23082935a9ab78c6a0d7f447d65:A 2024-11-16T10:26:03,147 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:26:03,148 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32733 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:26:03,149 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): b6c1e23082935a9ab78c6a0d7f447d65/C is initiating minor compaction (all files) 2024-11-16T10:26:03,149 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6c1e23082935a9ab78c6a0d7f447d65/C in TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:03,149 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/c73f352bc0d14621b2d0efe2415a7a7e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/c4a0e9f773394a908db5395b2094123b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/cf4c530d87eb45aeb6c7c60a875e41e0] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp, totalSize=32.0 K 2024-11-16T10:26:03,149 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting c73f352bc0d14621b2d0efe2415a7a7e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731752759291 2024-11-16T10:26:03,149 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6c1e23082935a9ab78c6a0d7f447d65/B of b6c1e23082935a9ab78c6a0d7f447d65 into d1c623593e934e059810f8903e1c42ab(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:26:03,149 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4a0e9f773394a908db5395b2094123b, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731752760380 2024-11-16T10:26:03,149 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:26:03,149 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., storeName=b6c1e23082935a9ab78c6a0d7f447d65/B, priority=13, startTime=1731752762715; duration=0sec 2024-11-16T10:26:03,149 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:03,149 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6c1e23082935a9ab78c6a0d7f447d65:B 2024-11-16T10:26:03,150 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf4c530d87eb45aeb6c7c60a875e41e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731752761454 2024-11-16T10:26:03,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/2f10033cd67c47b499ba21d27144bead is 50, key is test_row_0/B:col10/1731752762716/Put/seqid=0 2024-11-16T10:26:03,155 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6c1e23082935a9ab78c6a0d7f447d65#C#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:26:03,155 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/33a627055d3c4814bf46667d2a0cb494 is 50, key is test_row_0/C:col10/1731752761468/Put/seqid=0 2024-11-16T10:26:03,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742309_1485 (size=12301) 2024-11-16T10:26:03,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742310_1486 (size=13119) 2024-11-16T10:26:03,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T10:26:03,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:03,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:26:03,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:03,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:03,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:03,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752823376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:03,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:03,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:03,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:03,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752823387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:03,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:03,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:03,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:03,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T10:26:03,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:03,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:26:03,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:03,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:03,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:03,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:03,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/2f10033cd67c47b499ba21d27144bead 2024-11-16T10:26:03,569 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/33a627055d3c4814bf46667d2a0cb494 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/33a627055d3c4814bf46667d2a0cb494 2024-11-16T10:26:03,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/edbd1852d1bd4053a3a9400cdcccf0ca is 50, key is test_row_0/C:col10/1731752762716/Put/seqid=0 2024-11-16T10:26:03,573 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6c1e23082935a9ab78c6a0d7f447d65/C of b6c1e23082935a9ab78c6a0d7f447d65 into 33a627055d3c4814bf46667d2a0cb494(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:26:03,573 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:26:03,573 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., storeName=b6c1e23082935a9ab78c6a0d7f447d65/C, priority=13, startTime=1731752762715; duration=0sec 2024-11-16T10:26:03,574 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:03,574 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6c1e23082935a9ab78c6a0d7f447d65:C 2024-11-16T10:26:03,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742311_1487 (size=12301) 2024-11-16T10:26:03,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T10:26:03,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:03,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:26:03,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:03,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:03,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:03,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:03,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:03,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:03,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38862 deadline: 1731752823726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:03,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38888 deadline: 1731752823726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:03,727 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_2' in b6c1e23082935a9ab78c6a0d7f447d65 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:26:03,727 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_1' in b6c1e23082935a9ab78c6a0d7f447d65 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:26:03,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:03,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:03,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:03,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:03,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:03,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:03,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T10:26:03,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:03,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:26:03,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:03,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:03,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:03,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:03,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:03,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731752823887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:03,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:03,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:03,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:03,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:03,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38906 deadline: 1731752823897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:03,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 , the old value is region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:03,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:03,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., hostname=b16ff38458c1,35873,1731752627506, seqNum=2 because the exception is null or not the one we care about 2024-11-16T10:26:03,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T10:26:03,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:03,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:26:03,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:03,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:03,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:03,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:03,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T10:26:03,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/edbd1852d1bd4053a3a9400cdcccf0ca 2024-11-16T10:26:03,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/b5739ddb7cbe472bac406efcba40d2b2 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/b5739ddb7cbe472bac406efcba40d2b2 2024-11-16T10:26:03,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/b5739ddb7cbe472bac406efcba40d2b2, entries=250, sequenceid=439, filesize=16.8 K 2024-11-16T10:26:03,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/2f10033cd67c47b499ba21d27144bead as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/2f10033cd67c47b499ba21d27144bead 2024-11-16T10:26:03,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/2f10033cd67c47b499ba21d27144bead, entries=150, sequenceid=439, filesize=12.0 K 2024-11-16T10:26:03,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/edbd1852d1bd4053a3a9400cdcccf0ca as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/edbd1852d1bd4053a3a9400cdcccf0ca 2024-11-16T10:26:03,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/edbd1852d1bd4053a3a9400cdcccf0ca, entries=150, sequenceid=439, filesize=12.0 K 2024-11-16T10:26:04,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for b6c1e23082935a9ab78c6a0d7f447d65 in 1282ms, sequenceid=439, compaction requested=false 2024-11-16T10:26:04,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:26:04,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T10:26:04,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:04,057 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing b6c1e23082935a9ab78c6a0d7f447d65 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-16T10:26:04,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/83393851ff6b460287996af1b4da1467 is 50, key is test_row_0/A:col10/1731752762727/Put/seqid=0 2024-11-16T10:26:04,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742312_1488 (size=12301) 2024-11-16T10:26:04,465 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/83393851ff6b460287996af1b4da1467 2024-11-16T10:26:04,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/c5e94f46c5a446158f03c25cef3a1519 is 50, key is test_row_0/B:col10/1731752762727/Put/seqid=0 2024-11-16T10:26:04,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742313_1489 (size=12301) 2024-11-16T10:26:04,874 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/c5e94f46c5a446158f03c25cef3a1519 2024-11-16T10:26:04,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/6b0f5075646e47d8bc638a2301af9b36 is 50, key is test_row_0/C:col10/1731752762727/Put/seqid=0 2024-11-16T10:26:04,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742314_1490 (size=12301) 2024-11-16T10:26:04,898 INFO [Thread-1784 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1784. 2024-11-16T10:26:04,898 DEBUG [Thread-1784 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:26:04,898 DEBUG [Thread-1784 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:04,898 DEBUG [Thread-1784 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:04,898 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:26:04,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. as already flushing 2024-11-16T10:26:04,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:26:04,907 INFO [Thread-1788 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1788. 2024-11-16T10:26:04,908 DEBUG [Thread-1788 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:26:04,908 DEBUG [Thread-1788 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:04,908 DEBUG [Thread-1788 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:04,908 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:26:05,285 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/6b0f5075646e47d8bc638a2301af9b36 2024-11-16T10:26:05,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/83393851ff6b460287996af1b4da1467 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/83393851ff6b460287996af1b4da1467 2024-11-16T10:26:05,297 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/83393851ff6b460287996af1b4da1467, entries=150, sequenceid=451, filesize=12.0 K 2024-11-16T10:26:05,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/c5e94f46c5a446158f03c25cef3a1519 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/c5e94f46c5a446158f03c25cef3a1519 2024-11-16T10:26:05,302 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/c5e94f46c5a446158f03c25cef3a1519, entries=150, sequenceid=451, filesize=12.0 K 2024-11-16T10:26:05,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/6b0f5075646e47d8bc638a2301af9b36 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/6b0f5075646e47d8bc638a2301af9b36 2024-11-16T10:26:05,306 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/6b0f5075646e47d8bc638a2301af9b36, entries=150, sequenceid=451, filesize=12.0 K 2024-11-16T10:26:05,306 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=13.42 KB/13740 for b6c1e23082935a9ab78c6a0d7f447d65 in 1249ms, sequenceid=451, compaction requested=true 2024-11-16T10:26:05,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for b6c1e23082935a9ab78c6a0d7f447d65: 2024-11-16T10:26:05,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:05,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-16T10:26:05,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-16T10:26:05,309 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-16T10:26:05,309 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5500 sec 2024-11-16T10:26:05,311 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 3.5540 sec 2024-11-16T10:26:05,737 INFO [Thread-1782 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1782. 2024-11-16T10:26:05,737 DEBUG [Thread-1782 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:26:05,737 DEBUG [Thread-1782 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:05,738 DEBUG [Thread-1782 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:05,738 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:26:05,752 INFO [Thread-1790 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1790. 2024-11-16T10:26:05,752 DEBUG [Thread-1790 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:26:05,752 DEBUG [Thread-1790 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:05,752 DEBUG [Thread-1790 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:05,752 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:26:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T10:26:05,926 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:26:06,448 INFO [Thread-1786 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1786. 2024-11-16T10:26:06,448 DEBUG [Thread-1786 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:26:06,448 DEBUG [Thread-1786 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:06,448 DEBUG [Thread-1786 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:06,448 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:26:06,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T10:26:06,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-16T10:26:06,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-11-16T10:26:06,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-16T10:26:06,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-16T10:26:06,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-16T10:26:06,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T10:26:06,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T10:26:06,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1499 2024-11-16T10:26:06,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4494 rows 2024-11-16T10:26:06,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1513 2024-11-16T10:26:06,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4538 rows 2024-11-16T10:26:06,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1511 2024-11-16T10:26:06,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4522 rows 2024-11-16T10:26:06,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1507 2024-11-16T10:26:06,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4521 rows 2024-11-16T10:26:06,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1509 2024-11-16T10:26:06,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4527 rows 2024-11-16T10:26:06,449 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T10:26:06,449 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T10:26:06,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:06,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:06,449 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:26:06,450 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T10:26:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T10:26:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T10:26:06,453 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752766453"}]},"ts":"1731752766453"} 2024-11-16T10:26:06,454 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T10:26:06,454 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T10:26:06,455 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T10:26:06,456 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6c1e23082935a9ab78c6a0d7f447d65, UNASSIGN}] 2024-11-16T10:26:06,457 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6c1e23082935a9ab78c6a0d7f447d65, UNASSIGN 2024-11-16T10:26:06,457 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=b6c1e23082935a9ab78c6a0d7f447d65, regionState=CLOSING, regionLocation=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:06,459 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6c1e23082935a9ab78c6a0d7f447d65, UNASSIGN because future has completed 2024-11-16T10:26:06,459 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T10:26:06,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506}] 2024-11-16T10:26:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T10:26:06,612 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:26:06,612 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T10:26:06,612 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing b6c1e23082935a9ab78c6a0d7f447d65, disabling compactions & flushes 2024-11-16T10:26:06,612 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:06,612 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:06,612 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. after waiting 0 ms 2024-11-16T10:26:06,612 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:06,612 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(2902): Flushing b6c1e23082935a9ab78c6a0d7f447d65 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-16T10:26:06,615 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/be7aa91833174c6897c064ae1840c187 is 50, key is test_row_0/A:col10/1731752766447/Put/seqid=0 2024-11-16T10:26:06,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742315_1491 (size=12301) 2024-11-16T10:26:06,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T10:26:07,019 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/be7aa91833174c6897c064ae1840c187 2024-11-16T10:26:07,024 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/c85e8a883a0a4168b44762a9553de8b3 is 50, key is test_row_0/B:col10/1731752766447/Put/seqid=0 2024-11-16T10:26:07,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742316_1492 (size=12301) 2024-11-16T10:26:07,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T10:26:07,427 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/c85e8a883a0a4168b44762a9553de8b3 2024-11-16T10:26:07,434 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/d49a4befe1f24fe4858515d1486f0fb4 is 50, key is test_row_0/C:col10/1731752766447/Put/seqid=0 2024-11-16T10:26:07,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742317_1493 (size=12301) 2024-11-16T10:26:07,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T10:26:07,839 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/d49a4befe1f24fe4858515d1486f0fb4 2024-11-16T10:26:07,843 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/A/be7aa91833174c6897c064ae1840c187 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/be7aa91833174c6897c064ae1840c187 2024-11-16T10:26:07,846 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/be7aa91833174c6897c064ae1840c187, entries=150, sequenceid=459, filesize=12.0 K 2024-11-16T10:26:07,846 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/B/c85e8a883a0a4168b44762a9553de8b3 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/c85e8a883a0a4168b44762a9553de8b3 2024-11-16T10:26:07,849 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/c85e8a883a0a4168b44762a9553de8b3, entries=150, sequenceid=459, filesize=12.0 K 2024-11-16T10:26:07,850 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/.tmp/C/d49a4befe1f24fe4858515d1486f0fb4 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/d49a4befe1f24fe4858515d1486f0fb4 2024-11-16T10:26:07,853 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/d49a4befe1f24fe4858515d1486f0fb4, entries=150, sequenceid=459, filesize=12.0 K 2024-11-16T10:26:07,853 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for b6c1e23082935a9ab78c6a0d7f447d65 in 1241ms, sequenceid=459, compaction requested=true 2024-11-16T10:26:07,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/e5830dcfe5cd4bd08b1c11eb36cc9a9e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/28a98b91d4e4458db38252388a992403, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/8868d0fb8c924042b0d80a0be649cb63, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/63731870bc11466580687784b0bca801, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/87ccfd6286ea46f98065ad24083f16f4, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/46f50474e257447abb583af4e987dc0e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/3f40159007cc4a119d168a44778aa8fd, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/972c9b733d0f4d9ca0a8898a72de20c3, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/fa491e5a1ab8477c8785e0315314fa5a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/f5eeec26763b46658dc5dfc4d88b096e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/ca243ad30a1542dda81038a3baf58b97, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/c8f75f7682e742b59593764c8422bd11, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/98a86ed6dabf4f748715ce4643d1376e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/4d8e9fb1be55429ea515ff8b59e7b6cc, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/ad03fd83403d46978b1122f33937916b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/e25aed6702ae41e9867c39a99170e47a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/339537d3dc99465994d3a06700f741cb, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/28bff8e5815149a6bd079446b0f716ec, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/5175b4c863f5450eb4bb5c4572bfc78b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/44bb40ed60c74a8fbce7385448940b4b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/039e2bf1d51040f2a76cb080a2e59938, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/58b3d40c34e44800a8b6aae0bcb3df4b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/a3aa4eb5ca2946528b14516333c88dd4, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/00bcde5576a649f0aba9d063f3a5f9f3] to archive 2024-11-16T10:26:07,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:26:07,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/e5830dcfe5cd4bd08b1c11eb36cc9a9e to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/e5830dcfe5cd4bd08b1c11eb36cc9a9e 2024-11-16T10:26:07,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/28a98b91d4e4458db38252388a992403 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/28a98b91d4e4458db38252388a992403 2024-11-16T10:26:07,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/8868d0fb8c924042b0d80a0be649cb63 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/8868d0fb8c924042b0d80a0be649cb63 2024-11-16T10:26:07,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/63731870bc11466580687784b0bca801 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/63731870bc11466580687784b0bca801 2024-11-16T10:26:07,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/87ccfd6286ea46f98065ad24083f16f4 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/87ccfd6286ea46f98065ad24083f16f4 2024-11-16T10:26:07,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/46f50474e257447abb583af4e987dc0e to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/46f50474e257447abb583af4e987dc0e 2024-11-16T10:26:07,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/3f40159007cc4a119d168a44778aa8fd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/3f40159007cc4a119d168a44778aa8fd 2024-11-16T10:26:07,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/972c9b733d0f4d9ca0a8898a72de20c3 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/972c9b733d0f4d9ca0a8898a72de20c3 2024-11-16T10:26:07,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/fa491e5a1ab8477c8785e0315314fa5a to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/fa491e5a1ab8477c8785e0315314fa5a 2024-11-16T10:26:07,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/f5eeec26763b46658dc5dfc4d88b096e to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/f5eeec26763b46658dc5dfc4d88b096e 2024-11-16T10:26:07,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/ca243ad30a1542dda81038a3baf58b97 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/ca243ad30a1542dda81038a3baf58b97 2024-11-16T10:26:07,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/c8f75f7682e742b59593764c8422bd11 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/c8f75f7682e742b59593764c8422bd11 2024-11-16T10:26:07,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/98a86ed6dabf4f748715ce4643d1376e to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/98a86ed6dabf4f748715ce4643d1376e 2024-11-16T10:26:07,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/4d8e9fb1be55429ea515ff8b59e7b6cc to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/4d8e9fb1be55429ea515ff8b59e7b6cc 2024-11-16T10:26:07,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/ad03fd83403d46978b1122f33937916b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/ad03fd83403d46978b1122f33937916b 2024-11-16T10:26:07,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/e25aed6702ae41e9867c39a99170e47a to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/e25aed6702ae41e9867c39a99170e47a 2024-11-16T10:26:07,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/339537d3dc99465994d3a06700f741cb to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/339537d3dc99465994d3a06700f741cb 2024-11-16T10:26:07,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/28bff8e5815149a6bd079446b0f716ec to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/28bff8e5815149a6bd079446b0f716ec 2024-11-16T10:26:07,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/5175b4c863f5450eb4bb5c4572bfc78b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/5175b4c863f5450eb4bb5c4572bfc78b 2024-11-16T10:26:07,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/44bb40ed60c74a8fbce7385448940b4b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/44bb40ed60c74a8fbce7385448940b4b 2024-11-16T10:26:07,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/039e2bf1d51040f2a76cb080a2e59938 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/039e2bf1d51040f2a76cb080a2e59938 2024-11-16T10:26:07,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/58b3d40c34e44800a8b6aae0bcb3df4b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/58b3d40c34e44800a8b6aae0bcb3df4b 2024-11-16T10:26:07,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/a3aa4eb5ca2946528b14516333c88dd4 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/a3aa4eb5ca2946528b14516333c88dd4 2024-11-16T10:26:07,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/00bcde5576a649f0aba9d063f3a5f9f3 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/00bcde5576a649f0aba9d063f3a5f9f3 2024-11-16T10:26:07,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/4faa3b82741d481fae1a3d1d4b9b76f2, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/4882ca00a39544e2aebdcc5c05d4e786, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/b73e05e5cf38491da19e19e4d5a87f44, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/15ab4e4587db4fb98c3a969b11b42f3d, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/53b91319707744ba87467950605277d9, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/8db3687c242946ad90c25bacba67b7ac, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/f24e67a60c6642aba03ed6f0ab1824fb, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/ebe990d451f3423a9297551cfad75004, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/dd7fe84286e44fcca7bae25fe30a57c0, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/9d21225df6a844a7911f3ff15b942975, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/7f2a4cdf5ee4427eac25d5d61c42a2d0, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/f157992dd90a4984b695529c990be5d7, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/18dea4c60b8c47f085fce71c7ecc47f9, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/32896a0631994f7fb75deaf122fac073, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/aef5659e4fbf47ecb7d1500feb4740c5, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/267550b3d8404a3d95bc0338163e5f7f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/44c5966d002f43458c271176f71138f3, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/b6fe3f8e654244419ca6fa8dace58f40, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/229bf036feb14a29b17d981fd946d593, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/c4441e36bac44e8694209ff81c8f57c3, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/8e06e1a4f319450581116278380dba83, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/586e94aa73b94b9f9b14a909ee558ccc, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/9528ff32c7d743b086f59f8a10f14a3b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/e5ab4f6948e44d499094925b65654baa] to archive 2024-11-16T10:26:07,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:26:07,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/4faa3b82741d481fae1a3d1d4b9b76f2 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/4faa3b82741d481fae1a3d1d4b9b76f2 2024-11-16T10:26:07,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/4882ca00a39544e2aebdcc5c05d4e786 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/4882ca00a39544e2aebdcc5c05d4e786 2024-11-16T10:26:07,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/b73e05e5cf38491da19e19e4d5a87f44 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/b73e05e5cf38491da19e19e4d5a87f44 2024-11-16T10:26:07,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/15ab4e4587db4fb98c3a969b11b42f3d to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/15ab4e4587db4fb98c3a969b11b42f3d 2024-11-16T10:26:07,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/53b91319707744ba87467950605277d9 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/53b91319707744ba87467950605277d9 2024-11-16T10:26:07,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/8db3687c242946ad90c25bacba67b7ac to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/8db3687c242946ad90c25bacba67b7ac 2024-11-16T10:26:07,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/f24e67a60c6642aba03ed6f0ab1824fb to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/f24e67a60c6642aba03ed6f0ab1824fb 2024-11-16T10:26:07,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/ebe990d451f3423a9297551cfad75004 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/ebe990d451f3423a9297551cfad75004 2024-11-16T10:26:07,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/dd7fe84286e44fcca7bae25fe30a57c0 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/dd7fe84286e44fcca7bae25fe30a57c0 2024-11-16T10:26:07,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/9d21225df6a844a7911f3ff15b942975 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/9d21225df6a844a7911f3ff15b942975 2024-11-16T10:26:07,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/7f2a4cdf5ee4427eac25d5d61c42a2d0 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/7f2a4cdf5ee4427eac25d5d61c42a2d0 2024-11-16T10:26:07,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/f157992dd90a4984b695529c990be5d7 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/f157992dd90a4984b695529c990be5d7 2024-11-16T10:26:07,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/18dea4c60b8c47f085fce71c7ecc47f9 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/18dea4c60b8c47f085fce71c7ecc47f9 2024-11-16T10:26:07,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/32896a0631994f7fb75deaf122fac073 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/32896a0631994f7fb75deaf122fac073 2024-11-16T10:26:07,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/aef5659e4fbf47ecb7d1500feb4740c5 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/aef5659e4fbf47ecb7d1500feb4740c5 2024-11-16T10:26:07,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/267550b3d8404a3d95bc0338163e5f7f to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/267550b3d8404a3d95bc0338163e5f7f 2024-11-16T10:26:07,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/44c5966d002f43458c271176f71138f3 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/44c5966d002f43458c271176f71138f3 2024-11-16T10:26:07,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/b6fe3f8e654244419ca6fa8dace58f40 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/b6fe3f8e654244419ca6fa8dace58f40 2024-11-16T10:26:07,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/229bf036feb14a29b17d981fd946d593 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/229bf036feb14a29b17d981fd946d593 2024-11-16T10:26:07,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/c4441e36bac44e8694209ff81c8f57c3 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/c4441e36bac44e8694209ff81c8f57c3 2024-11-16T10:26:07,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/8e06e1a4f319450581116278380dba83 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/8e06e1a4f319450581116278380dba83 2024-11-16T10:26:07,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/586e94aa73b94b9f9b14a909ee558ccc to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/586e94aa73b94b9f9b14a909ee558ccc 2024-11-16T10:26:07,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/9528ff32c7d743b086f59f8a10f14a3b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/9528ff32c7d743b086f59f8a10f14a3b 2024-11-16T10:26:07,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/e5ab4f6948e44d499094925b65654baa to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/e5ab4f6948e44d499094925b65654baa 2024-11-16T10:26:07,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/057cdc5828664fc68e9af5f8e58ce4b8, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/d72ee1f1696b42a28c5947a097baca17, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/a466d20f7ac94ad6a0761339f83067c2, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/2b7f53029c3a4d8db4ce30a7bac4326a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/93deb7b49b944888a48ad611124af178, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/481f3b9917aa4ea28e9ad107fe3b42c9, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/c3699d82e8d44cbdbe4a9e4770b39102, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/2083e59881464f4d9e06355f63593ab2, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/4b98c830d5ab4d37a025c896e2044257, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/2aea9100cc164a688e9052919b7111d6, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/fcf2f30bb8c14e95ba86f9194c1cd861, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/6a15ec989f3a4944bbad7d99bebae30a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/03105004e5ac4d2faf31b253fdffcc69, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/5309b62c3dfb4b49aebc33eefdc6dd1b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/b5e738d93a8a46138aecff1d8252dfc2, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/e2feaeb60a374f60892dcf7ac7ae46f0, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/db8262ca3b474e818aba6c82ca852042, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/0ad753e11b7744c18d7f02318a658cae, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/ebecbe76d92541299ed5f248d189dcc3, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/95bb7cde876e4522b38fe065ec44a431, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/c73f352bc0d14621b2d0efe2415a7a7e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/aba3eb2464024701996494c451beb75a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/c4a0e9f773394a908db5395b2094123b, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/cf4c530d87eb45aeb6c7c60a875e41e0] to archive 2024-11-16T10:26:07,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:26:07,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/057cdc5828664fc68e9af5f8e58ce4b8 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/057cdc5828664fc68e9af5f8e58ce4b8 2024-11-16T10:26:07,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/d72ee1f1696b42a28c5947a097baca17 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/d72ee1f1696b42a28c5947a097baca17 2024-11-16T10:26:07,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/a466d20f7ac94ad6a0761339f83067c2 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/a466d20f7ac94ad6a0761339f83067c2 2024-11-16T10:26:07,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/2b7f53029c3a4d8db4ce30a7bac4326a to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/2b7f53029c3a4d8db4ce30a7bac4326a 2024-11-16T10:26:07,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/93deb7b49b944888a48ad611124af178 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/93deb7b49b944888a48ad611124af178 2024-11-16T10:26:07,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/481f3b9917aa4ea28e9ad107fe3b42c9 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/481f3b9917aa4ea28e9ad107fe3b42c9 2024-11-16T10:26:07,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/c3699d82e8d44cbdbe4a9e4770b39102 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/c3699d82e8d44cbdbe4a9e4770b39102 2024-11-16T10:26:07,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/2083e59881464f4d9e06355f63593ab2 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/2083e59881464f4d9e06355f63593ab2 2024-11-16T10:26:07,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/4b98c830d5ab4d37a025c896e2044257 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/4b98c830d5ab4d37a025c896e2044257 2024-11-16T10:26:07,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/2aea9100cc164a688e9052919b7111d6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/2aea9100cc164a688e9052919b7111d6 2024-11-16T10:26:07,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/fcf2f30bb8c14e95ba86f9194c1cd861 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/fcf2f30bb8c14e95ba86f9194c1cd861 2024-11-16T10:26:07,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/6a15ec989f3a4944bbad7d99bebae30a to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/6a15ec989f3a4944bbad7d99bebae30a 2024-11-16T10:26:07,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/03105004e5ac4d2faf31b253fdffcc69 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/03105004e5ac4d2faf31b253fdffcc69 2024-11-16T10:26:07,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/5309b62c3dfb4b49aebc33eefdc6dd1b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/5309b62c3dfb4b49aebc33eefdc6dd1b 2024-11-16T10:26:07,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/b5e738d93a8a46138aecff1d8252dfc2 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/b5e738d93a8a46138aecff1d8252dfc2 2024-11-16T10:26:07,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/e2feaeb60a374f60892dcf7ac7ae46f0 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/e2feaeb60a374f60892dcf7ac7ae46f0 2024-11-16T10:26:07,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/db8262ca3b474e818aba6c82ca852042 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/db8262ca3b474e818aba6c82ca852042 2024-11-16T10:26:07,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/0ad753e11b7744c18d7f02318a658cae to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/0ad753e11b7744c18d7f02318a658cae 2024-11-16T10:26:07,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/ebecbe76d92541299ed5f248d189dcc3 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/ebecbe76d92541299ed5f248d189dcc3 2024-11-16T10:26:07,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/95bb7cde876e4522b38fe065ec44a431 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/95bb7cde876e4522b38fe065ec44a431 2024-11-16T10:26:07,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/c73f352bc0d14621b2d0efe2415a7a7e to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/c73f352bc0d14621b2d0efe2415a7a7e 2024-11-16T10:26:07,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/aba3eb2464024701996494c451beb75a to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/aba3eb2464024701996494c451beb75a 2024-11-16T10:26:07,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/c4a0e9f773394a908db5395b2094123b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/c4a0e9f773394a908db5395b2094123b 2024-11-16T10:26:07,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/cf4c530d87eb45aeb6c7c60a875e41e0 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/cf4c530d87eb45aeb6c7c60a875e41e0 2024-11-16T10:26:07,919 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/recovered.edits/462.seqid, newMaxSeqId=462, maxSeqId=1 2024-11-16T10:26:07,920 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. 2024-11-16T10:26:07,920 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for b6c1e23082935a9ab78c6a0d7f447d65: Waiting for close lock at 1731752766612Running coprocessor pre-close hooks at 1731752766612Disabling compacts and flushes for region at 1731752766612Disabling writes for close at 1731752766612Obtaining lock to block concurrent updates at 1731752766612Preparing flush snapshotting stores in b6c1e23082935a9ab78c6a0d7f447d65 at 1731752766612Finished memstore snapshotting TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731752766612Flushing stores of TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65. at 1731752766613 (+1 ms)Flushing b6c1e23082935a9ab78c6a0d7f447d65/A: creating writer at 1731752766613Flushing b6c1e23082935a9ab78c6a0d7f447d65/A: appending metadata at 1731752766615 (+2 ms)Flushing b6c1e23082935a9ab78c6a0d7f447d65/A: closing flushed file at 1731752766615Flushing b6c1e23082935a9ab78c6a0d7f447d65/B: creating writer at 1731752767021 (+406 ms)Flushing b6c1e23082935a9ab78c6a0d7f447d65/B: appending metadata at 1731752767023 (+2 ms)Flushing b6c1e23082935a9ab78c6a0d7f447d65/B: closing flushed file at 1731752767023Flushing b6c1e23082935a9ab78c6a0d7f447d65/C: creating writer at 1731752767431 (+408 ms)Flushing b6c1e23082935a9ab78c6a0d7f447d65/C: appending metadata at 1731752767434 (+3 ms)Flushing b6c1e23082935a9ab78c6a0d7f447d65/C: closing flushed file at 1731752767434Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3add8dcb: reopening flushed file at 1731752767842 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c0104a5: reopening flushed file at 1731752767846 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7cb4b4c9: reopening flushed file at 1731752767849 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for b6c1e23082935a9ab78c6a0d7f447d65 in 1241ms, sequenceid=459, compaction requested=true at 1731752767853 (+4 ms)Writing region close event to WAL at 1731752767913 (+60 ms)Running coprocessor post-close hooks at 1731752767920 (+7 ms)Closed at 1731752767920 2024-11-16T10:26:07,921 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:26:07,922 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=b6c1e23082935a9ab78c6a0d7f447d65, regionState=CLOSED 2024-11-16T10:26:07,923 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 because future has completed 2024-11-16T10:26:07,924 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-16T10:26:07,924 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure b6c1e23082935a9ab78c6a0d7f447d65, server=b16ff38458c1,35873,1731752627506 in 1.4640 sec 2024-11-16T10:26:07,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-16T10:26:07,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6c1e23082935a9ab78c6a0d7f447d65, UNASSIGN in 1.4680 sec 2024-11-16T10:26:07,927 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-16T10:26:07,927 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4710 sec 2024-11-16T10:26:07,928 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752767928"}]},"ts":"1731752767928"} 2024-11-16T10:26:07,929 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T10:26:07,929 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T10:26:07,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4790 sec 2024-11-16T10:26:08,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T10:26:08,597 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:26:08,597 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T10:26:08,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:26:08,599 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:26:08,599 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=139, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:26:08,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T10:26:08,602 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:26:08,604 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A, FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B, FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C, FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/recovered.edits] 2024-11-16T10:26:08,606 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/83393851ff6b460287996af1b4da1467 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/83393851ff6b460287996af1b4da1467 2024-11-16T10:26:08,607 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/b5739ddb7cbe472bac406efcba40d2b2 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/b5739ddb7cbe472bac406efcba40d2b2 2024-11-16T10:26:08,607 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/be7aa91833174c6897c064ae1840c187 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/be7aa91833174c6897c064ae1840c187 2024-11-16T10:26:08,608 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/d83570b5acda413ea1ad0b1a4ff89b4b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/A/d83570b5acda413ea1ad0b1a4ff89b4b 2024-11-16T10:26:08,609 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/2f10033cd67c47b499ba21d27144bead to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/2f10033cd67c47b499ba21d27144bead 2024-11-16T10:26:08,610 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/c5e94f46c5a446158f03c25cef3a1519 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/c5e94f46c5a446158f03c25cef3a1519 2024-11-16T10:26:08,611 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/c85e8a883a0a4168b44762a9553de8b3 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/c85e8a883a0a4168b44762a9553de8b3 2024-11-16T10:26:08,612 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/d1c623593e934e059810f8903e1c42ab to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/B/d1c623593e934e059810f8903e1c42ab 2024-11-16T10:26:08,613 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/33a627055d3c4814bf46667d2a0cb494 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/33a627055d3c4814bf46667d2a0cb494 2024-11-16T10:26:08,614 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/6b0f5075646e47d8bc638a2301af9b36 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/6b0f5075646e47d8bc638a2301af9b36 2024-11-16T10:26:08,615 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/d49a4befe1f24fe4858515d1486f0fb4 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/d49a4befe1f24fe4858515d1486f0fb4 2024-11-16T10:26:08,616 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/edbd1852d1bd4053a3a9400cdcccf0ca to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/C/edbd1852d1bd4053a3a9400cdcccf0ca 2024-11-16T10:26:08,618 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/recovered.edits/462.seqid to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65/recovered.edits/462.seqid 2024-11-16T10:26:08,619 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/b6c1e23082935a9ab78c6a0d7f447d65 2024-11-16T10:26:08,619 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T10:26:08,621 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=139, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:26:08,623 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T10:26:08,625 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T10:26:08,626 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=139, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:26:08,626 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T10:26:08,626 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731752768626"}]},"ts":"9223372036854775807"} 2024-11-16T10:26:08,628 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T10:26:08,628 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b6c1e23082935a9ab78c6a0d7f447d65, NAME => 'TestAcidGuarantees,,1731752740686.b6c1e23082935a9ab78c6a0d7f447d65.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T10:26:08,628 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T10:26:08,629 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731752768628"}]},"ts":"9223372036854775807"} 2024-11-16T10:26:08,630 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T10:26:08,631 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=139, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:26:08,632 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 34 msec 2024-11-16T10:26:08,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T10:26:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T10:26:08,707 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:26:08,714 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=241 (was 242), OpenFileDescriptor=453 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=541 (was 497) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3843 (was 4053) 2024-11-16T10:26:08,722 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=241, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=541, ProcessCount=11, AvailableMemoryMB=3842 2024-11-16T10:26:08,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T10:26:08,723 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T10:26:08,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T10:26:08,725 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T10:26:08,725 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:26:08,725 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 140 2024-11-16T10:26:08,726 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T10:26:08,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-16T10:26:08,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742318_1494 (size=959) 2024-11-16T10:26:08,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-16T10:26:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-16T10:26:09,132 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7 2024-11-16T10:26:09,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742319_1495 (size=53) 2024-11-16T10:26:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-16T10:26:09,539 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:26:09,539 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 7636762808bf3c8f7a7fd5b283d2de9b, disabling compactions & flushes 2024-11-16T10:26:09,539 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:09,539 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:09,539 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. after waiting 0 ms 2024-11-16T10:26:09,539 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:09,539 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:09,539 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7636762808bf3c8f7a7fd5b283d2de9b: Waiting for close lock at 1731752769539Disabling compacts and flushes for region at 1731752769539Disabling writes for close at 1731752769539Writing region close event to WAL at 1731752769539Closed at 1731752769539 2024-11-16T10:26:09,540 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T10:26:09,540 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731752769540"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731752769540"}]},"ts":"1731752769540"} 2024-11-16T10:26:09,541 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T10:26:09,542 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T10:26:09,542 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752769542"}]},"ts":"1731752769542"} 2024-11-16T10:26:09,543 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T10:26:09,543 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7636762808bf3c8f7a7fd5b283d2de9b, ASSIGN}] 2024-11-16T10:26:09,544 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7636762808bf3c8f7a7fd5b283d2de9b, ASSIGN 2024-11-16T10:26:09,544 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7636762808bf3c8f7a7fd5b283d2de9b, ASSIGN; state=OFFLINE, location=b16ff38458c1,35873,1731752627506; forceNewPlan=false, retain=false 2024-11-16T10:26:09,695 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=7636762808bf3c8f7a7fd5b283d2de9b, regionState=OPENING, regionLocation=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:09,696 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7636762808bf3c8f7a7fd5b283d2de9b, ASSIGN because future has completed 2024-11-16T10:26:09,696 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506}] 2024-11-16T10:26:09,850 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:09,851 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7752): Opening region: {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} 2024-11-16T10:26:09,851 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:09,851 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:26:09,851 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7794): checking encryption for 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:09,851 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7797): checking classloading for 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:09,852 INFO [StoreOpener-7636762808bf3c8f7a7fd5b283d2de9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:09,854 INFO [StoreOpener-7636762808bf3c8f7a7fd5b283d2de9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7636762808bf3c8f7a7fd5b283d2de9b columnFamilyName A 2024-11-16T10:26:09,854 DEBUG [StoreOpener-7636762808bf3c8f7a7fd5b283d2de9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:26:09,854 INFO [StoreOpener-7636762808bf3c8f7a7fd5b283d2de9b-1 {}] regionserver.HStore(327): Store=7636762808bf3c8f7a7fd5b283d2de9b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:26:09,854 INFO [StoreOpener-7636762808bf3c8f7a7fd5b283d2de9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:09,855 INFO [StoreOpener-7636762808bf3c8f7a7fd5b283d2de9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7636762808bf3c8f7a7fd5b283d2de9b columnFamilyName B 2024-11-16T10:26:09,855 DEBUG [StoreOpener-7636762808bf3c8f7a7fd5b283d2de9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:26:09,855 INFO [StoreOpener-7636762808bf3c8f7a7fd5b283d2de9b-1 {}] regionserver.HStore(327): Store=7636762808bf3c8f7a7fd5b283d2de9b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:26:09,855 INFO [StoreOpener-7636762808bf3c8f7a7fd5b283d2de9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:09,856 INFO [StoreOpener-7636762808bf3c8f7a7fd5b283d2de9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7636762808bf3c8f7a7fd5b283d2de9b columnFamilyName C 2024-11-16T10:26:09,856 DEBUG [StoreOpener-7636762808bf3c8f7a7fd5b283d2de9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:26:09,857 INFO [StoreOpener-7636762808bf3c8f7a7fd5b283d2de9b-1 {}] regionserver.HStore(327): Store=7636762808bf3c8f7a7fd5b283d2de9b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:26:09,857 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1038): replaying wal for 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:09,857 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:09,858 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:09,858 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1048): stopping wal replay for 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:09,859 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1060): Cleaning up temporary data for 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:09,859 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T10:26:09,860 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1093): writing seq id for 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:09,862 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T10:26:09,862 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1114): Opened 7636762808bf3c8f7a7fd5b283d2de9b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65594074, jitterRate=-0.02257212996482849}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T10:26:09,862 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:09,863 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1006): Region open journal for 7636762808bf3c8f7a7fd5b283d2de9b: Running coprocessor pre-open hook at 1731752769851Writing region info on filesystem at 1731752769851Initializing all the Stores at 1731752769852 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752769852Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752769852Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752769852Cleaning up temporary data from old regions at 1731752769859 (+7 ms)Running coprocessor post-open hooks at 1731752769862 (+3 ms)Region opened successfully at 1731752769863 (+1 ms) 2024-11-16T10:26:09,864 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., pid=142, masterSystemTime=1731752769848 2024-11-16T10:26:09,865 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:09,865 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:09,866 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=7636762808bf3c8f7a7fd5b283d2de9b, regionState=OPEN, openSeqNum=2, regionLocation=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:09,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-16T10:26:09,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 because future has completed 2024-11-16T10:26:09,869 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-16T10:26:09,869 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure 7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 in 172 msec 2024-11-16T10:26:09,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-16T10:26:09,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7636762808bf3c8f7a7fd5b283d2de9b, ASSIGN in 326 msec 2024-11-16T10:26:09,871 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T10:26:09,871 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752769871"}]},"ts":"1731752769871"} 2024-11-16T10:26:09,872 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T10:26:09,873 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T10:26:09,875 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1500 sec 2024-11-16T10:26:10,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-16T10:26:10,876 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:26:10,877 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@185b2823, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:10,877 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:26:10,877 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:26:10,878 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:26:10,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:26:10,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:26:10,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4022ed21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:10,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:26:10,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:26:10,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:10,880 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:26:10,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62e9da4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:10,881 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:26:10,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:26:10,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:26:10,882 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:26:10,883 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T10:26:10,884 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is b16ff38458c1,38809,1731752626047 2024-11-16T10:26:10,884 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@debe491 2024-11-16T10:26:10,884 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T10:26:10,885 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T10:26:10,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T10:26:10,886 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T10:26:10,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-16T10:26:10,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742320_1496 (size=995) 2024-11-16T10:26:11,297 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-16T10:26:11,297 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-16T10:26:11,299 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T10:26:11,301 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7636762808bf3c8f7a7fd5b283d2de9b, REOPEN/MOVE}] 2024-11-16T10:26:11,301 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7636762808bf3c8f7a7fd5b283d2de9b, REOPEN/MOVE 2024-11-16T10:26:11,302 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=7636762808bf3c8f7a7fd5b283d2de9b, regionState=CLOSING, regionLocation=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:11,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7636762808bf3c8f7a7fd5b283d2de9b, REOPEN/MOVE because future has completed 2024-11-16T10:26:11,303 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T10:26:11,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506}] 2024-11-16T10:26:11,455 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:11,456 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T10:26:11,456 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing 7636762808bf3c8f7a7fd5b283d2de9b, disabling compactions & flushes 2024-11-16T10:26:11,456 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:11,456 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:11,456 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. after waiting 0 ms 2024-11-16T10:26:11,456 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:11,459 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T10:26:11,460 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:11,460 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for 7636762808bf3c8f7a7fd5b283d2de9b: Waiting for close lock at 1731752771456Running coprocessor pre-close hooks at 1731752771456Disabling compacts and flushes for region at 1731752771456Disabling writes for close at 1731752771456Writing region close event to WAL at 1731752771456Running coprocessor post-close hooks at 1731752771459 (+3 ms)Closed at 1731752771459 2024-11-16T10:26:11,460 WARN [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionServer(3299): Not adding moved region record: 7636762808bf3c8f7a7fd5b283d2de9b to self. 2024-11-16T10:26:11,461 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:11,462 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=7636762808bf3c8f7a7fd5b283d2de9b, regionState=CLOSED 2024-11-16T10:26:11,463 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 because future has completed 2024-11-16T10:26:11,465 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-16T10:26:11,465 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure 7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 in 161 msec 2024-11-16T10:26:11,465 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7636762808bf3c8f7a7fd5b283d2de9b, REOPEN/MOVE; state=CLOSED, location=b16ff38458c1,35873,1731752627506; forceNewPlan=false, retain=true 2024-11-16T10:26:11,616 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=7636762808bf3c8f7a7fd5b283d2de9b, regionState=OPENING, regionLocation=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:11,617 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7636762808bf3c8f7a7fd5b283d2de9b, REOPEN/MOVE because future has completed 2024-11-16T10:26:11,617 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506}] 2024-11-16T10:26:11,771 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:11,771 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7752): Opening region: {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} 2024-11-16T10:26:11,771 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:11,772 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T10:26:11,772 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7794): checking encryption for 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:11,772 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7797): checking classloading for 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:11,773 INFO [StoreOpener-7636762808bf3c8f7a7fd5b283d2de9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:11,774 INFO [StoreOpener-7636762808bf3c8f7a7fd5b283d2de9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7636762808bf3c8f7a7fd5b283d2de9b columnFamilyName A 2024-11-16T10:26:11,775 DEBUG [StoreOpener-7636762808bf3c8f7a7fd5b283d2de9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:26:11,775 INFO [StoreOpener-7636762808bf3c8f7a7fd5b283d2de9b-1 {}] regionserver.HStore(327): Store=7636762808bf3c8f7a7fd5b283d2de9b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:26:11,775 INFO [StoreOpener-7636762808bf3c8f7a7fd5b283d2de9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:11,776 INFO [StoreOpener-7636762808bf3c8f7a7fd5b283d2de9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7636762808bf3c8f7a7fd5b283d2de9b columnFamilyName B 2024-11-16T10:26:11,776 DEBUG [StoreOpener-7636762808bf3c8f7a7fd5b283d2de9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:26:11,776 INFO [StoreOpener-7636762808bf3c8f7a7fd5b283d2de9b-1 {}] regionserver.HStore(327): Store=7636762808bf3c8f7a7fd5b283d2de9b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:26:11,776 INFO [StoreOpener-7636762808bf3c8f7a7fd5b283d2de9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:11,777 INFO [StoreOpener-7636762808bf3c8f7a7fd5b283d2de9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7636762808bf3c8f7a7fd5b283d2de9b columnFamilyName C 2024-11-16T10:26:11,777 DEBUG [StoreOpener-7636762808bf3c8f7a7fd5b283d2de9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:26:11,777 INFO [StoreOpener-7636762808bf3c8f7a7fd5b283d2de9b-1 {}] regionserver.HStore(327): Store=7636762808bf3c8f7a7fd5b283d2de9b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T10:26:11,777 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1038): replaying wal for 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:11,778 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:11,778 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:11,779 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1048): stopping wal replay for 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:11,779 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1060): Cleaning up temporary data for 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:11,779 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T10:26:11,780 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1093): writing seq id for 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:11,781 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1114): Opened 7636762808bf3c8f7a7fd5b283d2de9b; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75334459, jitterRate=0.12257091701030731}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T10:26:11,781 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:11,782 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1006): Region open journal for 7636762808bf3c8f7a7fd5b283d2de9b: Running coprocessor pre-open hook at 1731752771772Writing region info on filesystem at 1731752771772Initializing all the Stores at 1731752771772Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752771772Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752771773 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731752771773Cleaning up temporary data from old regions at 1731752771779 (+6 ms)Running coprocessor post-open hooks at 1731752771781 (+2 ms)Region opened successfully at 1731752771781 2024-11-16T10:26:11,782 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., pid=147, masterSystemTime=1731752771769 2024-11-16T10:26:11,784 DEBUG [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:11,784 INFO [RS_OPEN_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:11,784 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=7636762808bf3c8f7a7fd5b283d2de9b, regionState=OPEN, openSeqNum=5, regionLocation=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:11,785 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 because future has completed 2024-11-16T10:26:11,788 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=145 2024-11-16T10:26:11,788 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure 7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 in 169 msec 2024-11-16T10:26:11,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-16T10:26:11,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7636762808bf3c8f7a7fd5b283d2de9b, REOPEN/MOVE in 487 msec 2024-11-16T10:26:11,791 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-16T10:26:11,791 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 491 msec 2024-11-16T10:26:11,793 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 905 msec 2024-11-16T10:26:11,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-16T10:26:11,794 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:26:11,794 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d29c9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,795 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:26:11,795 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:26:11,795 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:26:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:26:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:26:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@707b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:26:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:26:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:11,797 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:26:11,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1381af00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,798 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1572f078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,798 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:26:11,798 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:26:11,799 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:26:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:26:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:26:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1afef833, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:26:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:26:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:11,800 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:26:11,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e01de5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,801 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23448b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,801 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:26:11,801 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:26:11,802 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:26:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:26:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:26:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f93c61f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:26:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:26:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:11,803 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:26:11,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1373bd67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,804 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7db06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,804 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:26:11,804 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:26:11,804 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:26:11,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:26:11,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:26:11,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@696d56b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:26:11,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:26:11,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:11,806 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:26:11,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26c6a074, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,807 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ca8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,807 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:26:11,807 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:26:11,808 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:26:11,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:26:11,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:26:11,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29b9a0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:26:11,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:26:11,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:11,809 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:26:11,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e8be8cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,811 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5451f0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,811 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:26:11,811 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:26:11,811 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:26:11,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:26:11,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:26:11,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48ee106e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:26:11,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:26:11,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:11,813 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:26:11,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55718015, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,814 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e0fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,814 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:26:11,814 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:26:11,815 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:26:11,815 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:26:11,815 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:26:11,815 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@767e1e14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,815 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:26:11,815 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:26:11,815 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:11,816 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:26:11,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb03e98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbb8a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,817 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:26:11,817 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:26:11,818 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:26:11,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:26:11,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:26:11,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47dc2efe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:26:11,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:26:11,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:11,819 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:26:11,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24a77792, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,820 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a58f58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,820 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:26:11,820 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:26:11,821 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:26:11,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:26:11,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:26:11,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5569aec4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:26:11,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:26:11,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:11,822 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:26:11,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e6496e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,823 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e766e1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,823 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b16ff38458c1,38809,-1 for getting cluster id 2024-11-16T10:26:11,823 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T10:26:11,824 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '44c99309-bfed-483a-a470-cc4f96bfb762' 2024-11-16T10:26:11,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T10:26:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "44c99309-bfed-483a-a470-cc4f96bfb762" 2024-11-16T10:26:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f82ae71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b16ff38458c1,38809,-1] 2024-11-16T10:26:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T10:26:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:11,826 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T10:26:11,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36253b7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T10:26:11,827 DEBUG [Thread-2242 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:26:11,827 DEBUG [Thread-2244 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:26:11,827 DEBUG [Thread-2235 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:26:11,827 DEBUG [Thread-2233 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:26:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:26:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:26:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:26:11,828 DEBUG [Thread-2229 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:26:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:26:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:26:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:26:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:26:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:26:11,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:26:11,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:26:11,830 DEBUG [Thread-2237 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:26:11,830 DEBUG [Thread-2248 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:26:11,830 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:26:11,830 DEBUG [Thread-2231 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:26:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:26:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:26:11,831 DEBUG [Thread-2240 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:26:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-16T10:26:11,831 DEBUG [Thread-2246 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T10:26:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:26:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:26:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:26:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:26:11,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:26:11,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:26:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-16T10:26:11,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b16ff38458c1,35873,1731752627506, seqNum=-1] 2024-11-16T10:26:11,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T10:26:11,832 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:26:11,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:26:11,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:26:11,833 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:26:11,833 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:26:11,833 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:26:11,833 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:26:11,833 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:26:11,833 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:26:11,833 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:26:11,833 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T10:26:11,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5] 2024-11-16T10:26:11,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5] 2024-11-16T10:26:11,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5] 2024-11-16T10:26:11,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5] 2024-11-16T10:26:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5] 2024-11-16T10:26:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5] 2024-11-16T10:26:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5] 2024-11-16T10:26:11,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5] 2024-11-16T10:26:11,838 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5] 2024-11-16T10:26:11,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5] 2024-11-16T10:26:11,838 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:26:11,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:26:11,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:11,843 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:26:11,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7636762808bf3c8f7a7fd5b283d2de9b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T10:26:11,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c61e4f5c91a540ed86f70ca9a0e47045_7636762808bf3c8f7a7fd5b283d2de9b is 50, key is test_row_0/A:col10/1731752771837/Put/seqid=0 2024-11-16T10:26:11,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:11,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752831864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:11,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:11,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:11,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:11,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:11,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752831866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:11,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752831867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:11,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752831868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:11,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:11,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752831868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:11,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:11,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:11,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:11,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742321_1497 (size=12154) 2024-11-16T10:26:11,889 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:26:11,892 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c61e4f5c91a540ed86f70ca9a0e47045_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c61e4f5c91a540ed86f70ca9a0e47045_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:11,892 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/98b784610d31404998f549ae1ce511fb, store: [table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:11,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/98b784610d31404998f549ae1ce511fb is 175, key is test_row_0/A:col10/1731752771837/Put/seqid=0 2024-11-16T10:26:11,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742322_1498 (size=30955) 2024-11-16T10:26:11,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-16T10:26:11,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:11,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752831976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:11,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:11,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:11,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:11,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:11,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752831976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:11,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:11,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:11,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:11,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:11,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752831976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:11,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:11,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:11,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:11,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:11,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752831984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:11,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:11,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752831984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:11,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-16T10:26:11,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:11,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:11,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:11,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:11,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:11,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:12,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-16T10:26:12,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:12,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:12,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:12,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:12,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-16T10:26:12,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752832186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:12,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:12,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752832186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:12,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752832187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:12,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:12,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752832196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752832196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:12,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-16T10:26:12,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:12,298 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/98b784610d31404998f549ae1ce511fb 2024-11-16T10:26:12,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:12,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:12,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:12,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:12,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:12,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/65de4a495e9e4997ac5ee05687db122d is 50, key is test_row_0/B:col10/1731752771837/Put/seqid=0 2024-11-16T10:26:12,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742323_1499 (size=12001) 2024-11-16T10:26:12,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/65de4a495e9e4997ac5ee05687db122d 2024-11-16T10:26:12,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/a4c677056c0c4473a709ec0543e9e591 is 50, key is test_row_0/C:col10/1731752771837/Put/seqid=0 2024-11-16T10:26:12,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742324_1500 (size=12001) 2024-11-16T10:26:12,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/a4c677056c0c4473a709ec0543e9e591 2024-11-16T10:26:12,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/98b784610d31404998f549ae1ce511fb as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/98b784610d31404998f549ae1ce511fb 2024-11-16T10:26:12,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/98b784610d31404998f549ae1ce511fb, entries=150, sequenceid=16, filesize=30.2 K 2024-11-16T10:26:12,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/65de4a495e9e4997ac5ee05687db122d as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/65de4a495e9e4997ac5ee05687db122d 2024-11-16T10:26:12,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/65de4a495e9e4997ac5ee05687db122d, entries=150, sequenceid=16, filesize=11.7 K 2024-11-16T10:26:12,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/a4c677056c0c4473a709ec0543e9e591 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/a4c677056c0c4473a709ec0543e9e591 2024-11-16T10:26:12,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/a4c677056c0c4473a709ec0543e9e591, entries=150, sequenceid=16, filesize=11.7 K 2024-11-16T10:26:12,411 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 7636762808bf3c8f7a7fd5b283d2de9b in 568ms, sequenceid=16, compaction requested=false 2024-11-16T10:26:12,411 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-16T10:26:12,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:12,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-16T10:26:12,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:12,456 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 7636762808bf3c8f7a7fd5b283d2de9b 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T10:26:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-16T10:26:12,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c0e512abb404458e96343a7277de704c_7636762808bf3c8f7a7fd5b283d2de9b is 50, key is test_row_0/A:col10/1731752771860/Put/seqid=0 2024-11-16T10:26:12,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742325_1501 (size=12154) 2024-11-16T10:26:12,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:26:12,480 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c0e512abb404458e96343a7277de704c_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c0e512abb404458e96343a7277de704c_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:12,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/d2d70de2231641f4bec8355666552ffc, store: [table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:12,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/d2d70de2231641f4bec8355666552ffc is 175, key is test_row_0/A:col10/1731752771860/Put/seqid=0 2024-11-16T10:26:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:12,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:12,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742326_1502 (size=30955) 2024-11-16T10:26:12,508 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/d2d70de2231641f4bec8355666552ffc 2024-11-16T10:26:12,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/8b4e25b3b8ac4fcaa6cfac2b4360ef1a is 50, key is test_row_0/B:col10/1731752771860/Put/seqid=0 2024-11-16T10:26:12,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752832510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:12,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:12,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752832513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:12,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:12,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752832514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:12,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:12,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752832520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:12,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752832520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:12,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:12,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:12,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742327_1503 (size=12001) 2024-11-16T10:26:12,537 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/8b4e25b3b8ac4fcaa6cfac2b4360ef1a 2024-11-16T10:26:12,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/35248701bbbf4bdab1b890f858c2a8cd is 50, key is test_row_0/C:col10/1731752771860/Put/seqid=0 2024-11-16T10:26:12,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742328_1504 (size=12001) 2024-11-16T10:26:12,561 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/35248701bbbf4bdab1b890f858c2a8cd 2024-11-16T10:26:12,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/d2d70de2231641f4bec8355666552ffc as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/d2d70de2231641f4bec8355666552ffc 2024-11-16T10:26:12,567 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/d2d70de2231641f4bec8355666552ffc, entries=150, sequenceid=40, filesize=30.2 K 2024-11-16T10:26:12,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/8b4e25b3b8ac4fcaa6cfac2b4360ef1a as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/8b4e25b3b8ac4fcaa6cfac2b4360ef1a 2024-11-16T10:26:12,571 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/8b4e25b3b8ac4fcaa6cfac2b4360ef1a, entries=150, sequenceid=40, filesize=11.7 K 2024-11-16T10:26:12,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/35248701bbbf4bdab1b890f858c2a8cd as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/35248701bbbf4bdab1b890f858c2a8cd 2024-11-16T10:26:12,575 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/35248701bbbf4bdab1b890f858c2a8cd, entries=150, sequenceid=40, filesize=11.7 K 2024-11-16T10:26:12,576 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 7636762808bf3c8f7a7fd5b283d2de9b in 119ms, sequenceid=40, compaction requested=false 2024-11-16T10:26:12,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:12,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:12,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-16T10:26:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-16T10:26:12,581 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-16T10:26:12,581 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 741 msec 2024-11-16T10:26:12,583 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 752 msec 2024-11-16T10:26:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:12,627 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:26:12,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7636762808bf3c8f7a7fd5b283d2de9b 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T10:26:12,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e0ef16b3c98d433ea62b550fb39e752e_7636762808bf3c8f7a7fd5b283d2de9b is 50, key is test_row_0/A:col10/1731752772626/Put/seqid=0 2024-11-16T10:26:12,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742329_1505 (size=14594) 2024-11-16T10:26:12,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752832651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:12,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:12,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752832652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:12,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:12,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752832652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752832653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752832653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:12,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:12,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752832766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:12,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:12,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752832767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:12,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:12,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:12,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752832767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:12,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:12,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:12,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752832767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:12,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:12,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:12,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752832767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:12,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:12,771 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,771 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,771 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:12,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-16T10:26:12,967 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:26:12,967 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:26:12,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-16T10:26:12,969 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:26:12,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T10:26:12,969 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:26:12,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:26:12,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:12,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752832976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:12,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:12,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752832976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:12,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:12,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752832976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:12,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:12,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:12,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752832976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:12,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:12,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:12,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752832976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:12,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:12,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:13,044 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:26:13,047 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e0ef16b3c98d433ea62b550fb39e752e_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e0ef16b3c98d433ea62b550fb39e752e_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:13,048 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/11b8b65bf0c44f59b582277cced726dd, store: [table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:13,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/11b8b65bf0c44f59b582277cced726dd is 175, key is test_row_0/A:col10/1731752772626/Put/seqid=0 2024-11-16T10:26:13,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742330_1506 (size=39549) 2024-11-16T10:26:13,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T10:26:13,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:26:13,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:13,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:13,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:13,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:13,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:13,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:13,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:26:13,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:13,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:13,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:13,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:13,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:13,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:13,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T10:26:13,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:13,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752833286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:13,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:13,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752833287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:13,289 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:13,289 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:13,289 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:13,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:13,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:13,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:13,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:13,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752833287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:13,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:13,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:13,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:13,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:13,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752833287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:13,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:13,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:13,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:13,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:13,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752833287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:13,290 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:13,290 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:13,290 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:13,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:26:13,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:13,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:13,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:13,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:13,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:13,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:13,437 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T10:26:13,454 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/11b8b65bf0c44f59b582277cced726dd 2024-11-16T10:26:13,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/cd91782b76a34caebe2952a822ca94fd is 50, key is test_row_0/B:col10/1731752772626/Put/seqid=0 2024-11-16T10:26:13,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742331_1507 (size=12001) 2024-11-16T10:26:13,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:26:13,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:13,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:13,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:13,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:13,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:13,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:13,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T10:26:13,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:26:13,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:13,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:13,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:13,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:13,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:13,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:13,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:13,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752833796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:13,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:13,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752833796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:13,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:13,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:13,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:13,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:13,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:13,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:13,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752833796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:13,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:13,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:13,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:13,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:13,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752833796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:13,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:13,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:13,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:13,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:13,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:13,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752833796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:13,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:13,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:13,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:13,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/cd91782b76a34caebe2952a822ca94fd 2024-11-16T10:26:13,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/4e5c345fefff4059bdf295c402625a0e is 50, key is test_row_0/C:col10/1731752772626/Put/seqid=0 2024-11-16T10:26:13,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742332_1508 (size=12001) 2024-11-16T10:26:13,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:26:13,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:13,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:13,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:13,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:13,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:13,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:14,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:26:14,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:14,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:14,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:14,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:14,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:14,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T10:26:14,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:26:14,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:14,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:14,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:14,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:14,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:14,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:14,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/4e5c345fefff4059bdf295c402625a0e 2024-11-16T10:26:14,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/11b8b65bf0c44f59b582277cced726dd as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/11b8b65bf0c44f59b582277cced726dd 2024-11-16T10:26:14,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/11b8b65bf0c44f59b582277cced726dd, entries=200, sequenceid=54, filesize=38.6 K 2024-11-16T10:26:14,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/cd91782b76a34caebe2952a822ca94fd as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/cd91782b76a34caebe2952a822ca94fd 2024-11-16T10:26:14,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/cd91782b76a34caebe2952a822ca94fd, entries=150, sequenceid=54, filesize=11.7 K 2024-11-16T10:26:14,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/4e5c345fefff4059bdf295c402625a0e as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/4e5c345fefff4059bdf295c402625a0e 2024-11-16T10:26:14,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/4e5c345fefff4059bdf295c402625a0e, entries=150, sequenceid=54, filesize=11.7 K 2024-11-16T10:26:14,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 7636762808bf3c8f7a7fd5b283d2de9b in 1667ms, sequenceid=54, compaction requested=true 2024-11-16T10:26:14,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:14,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7636762808bf3c8f7a7fd5b283d2de9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:26:14,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:14,295 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:26:14,295 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:26:14,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7636762808bf3c8f7a7fd5b283d2de9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:26:14,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:14,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7636762808bf3c8f7a7fd5b283d2de9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:26:14,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:26:14,296 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:26:14,297 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): 7636762808bf3c8f7a7fd5b283d2de9b/B is initiating minor compaction (all files) 2024-11-16T10:26:14,297 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7636762808bf3c8f7a7fd5b283d2de9b/B in TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:14,297 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/65de4a495e9e4997ac5ee05687db122d, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/8b4e25b3b8ac4fcaa6cfac2b4360ef1a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/cd91782b76a34caebe2952a822ca94fd] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp, totalSize=35.2 K 2024-11-16T10:26:14,297 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:26:14,297 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): 7636762808bf3c8f7a7fd5b283d2de9b/A is initiating minor compaction (all files) 2024-11-16T10:26:14,297 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7636762808bf3c8f7a7fd5b283d2de9b/A in TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:14,297 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/98b784610d31404998f549ae1ce511fb, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/d2d70de2231641f4bec8355666552ffc, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/11b8b65bf0c44f59b582277cced726dd] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp, totalSize=99.1 K 2024-11-16T10:26:14,297 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:14,297 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. files: [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/98b784610d31404998f549ae1ce511fb, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/d2d70de2231641f4bec8355666552ffc, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/11b8b65bf0c44f59b582277cced726dd] 2024-11-16T10:26:14,298 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98b784610d31404998f549ae1ce511fb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731752771837 2024-11-16T10:26:14,298 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 65de4a495e9e4997ac5ee05687db122d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731752771837 2024-11-16T10:26:14,298 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2d70de2231641f4bec8355666552ffc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731752771860 2024-11-16T10:26:14,298 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b4e25b3b8ac4fcaa6cfac2b4360ef1a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731752771860 2024-11-16T10:26:14,298 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting cd91782b76a34caebe2952a822ca94fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731752772511 2024-11-16T10:26:14,298 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11b8b65bf0c44f59b582277cced726dd, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731752772502 2024-11-16T10:26:14,329 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:14,342 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7636762808bf3c8f7a7fd5b283d2de9b#B#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:26:14,342 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/8becd93d098149bba8ce74fb444c4ddf is 50, key is test_row_0/B:col10/1731752772626/Put/seqid=0 2024-11-16T10:26:14,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T10:26:14,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:14,343 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 7636762808bf3c8f7a7fd5b283d2de9b 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T10:26:14,355 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116bf570a48aabd4342b605743662952a27_7636762808bf3c8f7a7fd5b283d2de9b store=[table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:14,357 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116bf570a48aabd4342b605743662952a27_7636762808bf3c8f7a7fd5b283d2de9b, store=[table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:14,357 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116bf570a48aabd4342b605743662952a27_7636762808bf3c8f7a7fd5b283d2de9b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:14,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163c6037c036b44ac49ac7a87924bab792_7636762808bf3c8f7a7fd5b283d2de9b is 50, key is test_row_0/A:col10/1731752772652/Put/seqid=0 2024-11-16T10:26:14,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742333_1509 (size=12104) 2024-11-16T10:26:14,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742334_1510 (size=4469) 2024-11-16T10:26:14,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742335_1511 (size=12154) 2024-11-16T10:26:14,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:14,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:14,818 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/8becd93d098149bba8ce74fb444c4ddf as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/8becd93d098149bba8ce74fb444c4ddf 2024-11-16T10:26:14,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:14,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752834821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:14,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:14,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:14,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:14,824 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7636762808bf3c8f7a7fd5b283d2de9b/B of 7636762808bf3c8f7a7fd5b283d2de9b into 8becd93d098149bba8ce74fb444c4ddf(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:26:14,824 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:14,824 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., storeName=7636762808bf3c8f7a7fd5b283d2de9b/B, priority=13, startTime=1731752774295; duration=0sec 2024-11-16T10:26:14,824 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:26:14,824 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7636762808bf3c8f7a7fd5b283d2de9b:B 2024-11-16T10:26:14,825 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:26:14,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:14,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752834822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:14,825 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:26:14,825 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): 7636762808bf3c8f7a7fd5b283d2de9b/C is initiating minor compaction (all files) 2024-11-16T10:26:14,825 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7636762808bf3c8f7a7fd5b283d2de9b/C in TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:14,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:14,826 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/a4c677056c0c4473a709ec0543e9e591, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/35248701bbbf4bdab1b890f858c2a8cd, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/4e5c345fefff4059bdf295c402625a0e] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp, totalSize=35.2 K 2024-11-16T10:26:14,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:14,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:14,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752834822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:14,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:14,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:14,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:14,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:14,826 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting a4c677056c0c4473a709ec0543e9e591, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731752771837 2024-11-16T10:26:14,826 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 35248701bbbf4bdab1b890f858c2a8cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731752771860 2024-11-16T10:26:14,826 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e5c345fefff4059bdf295c402625a0e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731752772511 2024-11-16T10:26:14,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:14,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752834823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:14,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:14,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752834824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:14,831 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7636762808bf3c8f7a7fd5b283d2de9b#A#compaction#437 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:26:14,831 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/68545189aac84157b9d52afee5afe1d6 is 175, key is test_row_0/A:col10/1731752772626/Put/seqid=0 2024-11-16T10:26:14,844 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7636762808bf3c8f7a7fd5b283d2de9b#C#compaction#440 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:26:14,844 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/afce63b7ac4a4ca3a3c9f14ff9d5d860 is 50, key is test_row_0/C:col10/1731752772626/Put/seqid=0 2024-11-16T10:26:14,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742336_1512 (size=31058) 2024-11-16T10:26:14,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:26:14,860 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163c6037c036b44ac49ac7a87924bab792_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163c6037c036b44ac49ac7a87924bab792_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:14,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/161841f9206f47ca8de265820a6dd13e, store: [table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:14,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/161841f9206f47ca8de265820a6dd13e is 175, key is test_row_0/A:col10/1731752772652/Put/seqid=0 2024-11-16T10:26:14,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742337_1513 (size=12104) 2024-11-16T10:26:14,871 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/68545189aac84157b9d52afee5afe1d6 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/68545189aac84157b9d52afee5afe1d6 2024-11-16T10:26:14,874 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7636762808bf3c8f7a7fd5b283d2de9b/A of 7636762808bf3c8f7a7fd5b283d2de9b into 68545189aac84157b9d52afee5afe1d6(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:26:14,874 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:14,874 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., storeName=7636762808bf3c8f7a7fd5b283d2de9b/A, priority=13, startTime=1731752774295; duration=0sec 2024-11-16T10:26:14,874 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:14,874 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7636762808bf3c8f7a7fd5b283d2de9b:A 2024-11-16T10:26:14,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742338_1514 (size=30955) 2024-11-16T10:26:14,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:14,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752834926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:14,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:14,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:14,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:14,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752834936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:14,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:14,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:14,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:14,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752834936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:14,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:14,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:14,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:14,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:14,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752834937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:14,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:14,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:14,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:14,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:14,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752834937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:14,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:14,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:14,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:15,096 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T10:26:15,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T10:26:15,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:15,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752835136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:15,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:15,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752835146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:15,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:15,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752835146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:15,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:15,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:15,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752835146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:15,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:15,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752835146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:15,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:15,266 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/afce63b7ac4a4ca3a3c9f14ff9d5d860 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/afce63b7ac4a4ca3a3c9f14ff9d5d860 2024-11-16T10:26:15,269 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7636762808bf3c8f7a7fd5b283d2de9b/C of 7636762808bf3c8f7a7fd5b283d2de9b into afce63b7ac4a4ca3a3c9f14ff9d5d860(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:26:15,269 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:15,270 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., storeName=7636762808bf3c8f7a7fd5b283d2de9b/C, priority=13, startTime=1731752774295; duration=0sec 2024-11-16T10:26:15,270 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:15,270 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7636762808bf3c8f7a7fd5b283d2de9b:C 2024-11-16T10:26:15,289 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/161841f9206f47ca8de265820a6dd13e 2024-11-16T10:26:15,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/f78bc71134e74c1daec893554ed33a83 is 50, key is test_row_0/B:col10/1731752772652/Put/seqid=0 2024-11-16T10:26:15,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742339_1515 (size=12001) 2024-11-16T10:26:15,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752835446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:15,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:15,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:15,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752835456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:15,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:15,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:15,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752835456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:15,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752835456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:15,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:15,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752835456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:15,709 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/f78bc71134e74c1daec893554ed33a83 2024-11-16T10:26:15,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/4da28a9a4b4f412e825992c357a3e369 is 50, key is test_row_0/C:col10/1731752772652/Put/seqid=0 2024-11-16T10:26:15,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742340_1516 (size=12001) 2024-11-16T10:26:15,722 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/4da28a9a4b4f412e825992c357a3e369 2024-11-16T10:26:15,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/161841f9206f47ca8de265820a6dd13e as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/161841f9206f47ca8de265820a6dd13e 2024-11-16T10:26:15,730 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/161841f9206f47ca8de265820a6dd13e, entries=150, sequenceid=77, filesize=30.2 K 2024-11-16T10:26:15,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/f78bc71134e74c1daec893554ed33a83 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/f78bc71134e74c1daec893554ed33a83 2024-11-16T10:26:15,735 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/f78bc71134e74c1daec893554ed33a83, entries=150, sequenceid=77, filesize=11.7 K 2024-11-16T10:26:15,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/4da28a9a4b4f412e825992c357a3e369 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/4da28a9a4b4f412e825992c357a3e369 2024-11-16T10:26:15,740 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/4da28a9a4b4f412e825992c357a3e369, entries=150, sequenceid=77, filesize=11.7 K 2024-11-16T10:26:15,741 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 7636762808bf3c8f7a7fd5b283d2de9b in 1398ms, sequenceid=77, compaction requested=false 2024-11-16T10:26:15,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:15,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:15,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-16T10:26:15,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-16T10:26:15,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-16T10:26:15,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7720 sec 2024-11-16T10:26:15,746 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.7780 sec 2024-11-16T10:26:15,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:15,958 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:26:15,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7636762808bf3c8f7a7fd5b283d2de9b 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-16T10:26:15,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167adfb8c4dc5a4c5c9a806af29499f920_7636762808bf3c8f7a7fd5b283d2de9b is 50, key is test_row_0/A:col10/1731752774819/Put/seqid=0 2024-11-16T10:26:15,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752835984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:15,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752835985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:15,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752835986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:15,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:15,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752835988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:15,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:15,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:15,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752835991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:16,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742341_1517 (size=14594) 2024-11-16T10:26:16,006 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:26:16,008 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167adfb8c4dc5a4c5c9a806af29499f920_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167adfb8c4dc5a4c5c9a806af29499f920_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:16,009 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/f4fa580fb7b842579da583bc6de19956, store: [table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:16,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/f4fa580fb7b842579da583bc6de19956 is 175, key is test_row_0/A:col10/1731752774819/Put/seqid=0 2024-11-16T10:26:16,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742342_1518 (size=39549) 2024-11-16T10:26:16,029 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/f4fa580fb7b842579da583bc6de19956 2024-11-16T10:26:16,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/833d7f97e2b54668a6b590c81c94323f is 50, key is test_row_0/B:col10/1731752774819/Put/seqid=0 2024-11-16T10:26:16,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742343_1519 (size=12001) 2024-11-16T10:26:16,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:16,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752836097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:16,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:16,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752836097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:16,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:16,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752836097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:16,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:16,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:16,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752836100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:16,104 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,104 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,104 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:16,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:16,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752836106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:16,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:16,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752836306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:16,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:16,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752836307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:16,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:16,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752836307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752836307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:16,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:16,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752836316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:16,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:16,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/833d7f97e2b54668a6b590c81c94323f 2024-11-16T10:26:16,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/cc941b3146434341a7279ad4a58674a5 is 50, key is test_row_0/C:col10/1731752774819/Put/seqid=0 2024-11-16T10:26:16,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742344_1520 (size=12001) 2024-11-16T10:26:16,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:16,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752836616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:16,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:16,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:16,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752836616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:16,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:16,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:16,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752836616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:16,618 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:16,618 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,618 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:16,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752836616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:16,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:16,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:16,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752836626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:16,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/cc941b3146434341a7279ad4a58674a5 2024-11-16T10:26:16,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/f4fa580fb7b842579da583bc6de19956 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/f4fa580fb7b842579da583bc6de19956 2024-11-16T10:26:16,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/f4fa580fb7b842579da583bc6de19956, entries=200, sequenceid=94, filesize=38.6 K 2024-11-16T10:26:16,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/833d7f97e2b54668a6b590c81c94323f as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/833d7f97e2b54668a6b590c81c94323f 2024-11-16T10:26:16,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/833d7f97e2b54668a6b590c81c94323f, entries=150, sequenceid=94, filesize=11.7 K 2024-11-16T10:26:16,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/cc941b3146434341a7279ad4a58674a5 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/cc941b3146434341a7279ad4a58674a5 2024-11-16T10:26:16,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/cc941b3146434341a7279ad4a58674a5, entries=150, sequenceid=94, filesize=11.7 K 2024-11-16T10:26:16,898 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 7636762808bf3c8f7a7fd5b283d2de9b in 940ms, sequenceid=94, compaction requested=true 2024-11-16T10:26:16,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:16,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7636762808bf3c8f7a7fd5b283d2de9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:26:16,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:16,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7636762808bf3c8f7a7fd5b283d2de9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:26:16,898 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:26:16,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:16,898 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:26:16,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7636762808bf3c8f7a7fd5b283d2de9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:26:16,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:26:16,899 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:26:16,899 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): 7636762808bf3c8f7a7fd5b283d2de9b/B is initiating minor compaction (all files) 2024-11-16T10:26:16,900 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7636762808bf3c8f7a7fd5b283d2de9b/B in TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:16,900 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/8becd93d098149bba8ce74fb444c4ddf, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/f78bc71134e74c1daec893554ed33a83, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/833d7f97e2b54668a6b590c81c94323f] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp, totalSize=35.3 K 2024-11-16T10:26:16,900 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:26:16,900 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): 7636762808bf3c8f7a7fd5b283d2de9b/A is initiating minor compaction (all files) 2024-11-16T10:26:16,900 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7636762808bf3c8f7a7fd5b283d2de9b/A in TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:16,900 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 8becd93d098149bba8ce74fb444c4ddf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731752772511 2024-11-16T10:26:16,900 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/68545189aac84157b9d52afee5afe1d6, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/161841f9206f47ca8de265820a6dd13e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/f4fa580fb7b842579da583bc6de19956] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp, totalSize=99.2 K 2024-11-16T10:26:16,900 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:16,900 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. files: [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/68545189aac84157b9d52afee5afe1d6, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/161841f9206f47ca8de265820a6dd13e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/f4fa580fb7b842579da583bc6de19956] 2024-11-16T10:26:16,900 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting f78bc71134e74c1daec893554ed33a83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731752772651 2024-11-16T10:26:16,900 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68545189aac84157b9d52afee5afe1d6, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731752772511 2024-11-16T10:26:16,901 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 833d7f97e2b54668a6b590c81c94323f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731752774819 2024-11-16T10:26:16,901 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 161841f9206f47ca8de265820a6dd13e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731752772651 2024-11-16T10:26:16,901 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4fa580fb7b842579da583bc6de19956, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731752774815 2024-11-16T10:26:16,910 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7636762808bf3c8f7a7fd5b283d2de9b#B#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:26:16,911 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/e0caae33f41e4a4cad87152d44f25a88 is 50, key is test_row_0/B:col10/1731752774819/Put/seqid=0 2024-11-16T10:26:16,925 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:16,937 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116e50ff6275c7b4a19ac385608d9d6b37f_7636762808bf3c8f7a7fd5b283d2de9b store=[table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:16,940 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116e50ff6275c7b4a19ac385608d9d6b37f_7636762808bf3c8f7a7fd5b283d2de9b, store=[table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:16,940 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e50ff6275c7b4a19ac385608d9d6b37f_7636762808bf3c8f7a7fd5b283d2de9b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:16,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742345_1521 (size=12207) 2024-11-16T10:26:16,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742346_1522 (size=4469) 2024-11-16T10:26:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T10:26:17,127 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:26:17,127 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:26:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-16T10:26:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:17,129 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:26:17,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7636762808bf3c8f7a7fd5b283d2de9b 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T10:26:17,129 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:26:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T10:26:17,130 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:26:17,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:26:17,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169a59dab5729b487b9422e562f02cb784_7636762808bf3c8f7a7fd5b283d2de9b is 50, key is test_row_0/A:col10/1731752777128/Put/seqid=0 2024-11-16T10:26:17,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742347_1523 (size=12154) 2024-11-16T10:26:17,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:17,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752837143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:17,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:17,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:17,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752837147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:17,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:17,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:17,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752837148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:17,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752837149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:17,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752837150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:17,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T10:26:17,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:17,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752837256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:17,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752837256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:17,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:17,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752837266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:17,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752837266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:17,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:17,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752837266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:17,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-16T10:26:17,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:17,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:17,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:17,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:17,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:17,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:17,345 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/e0caae33f41e4a4cad87152d44f25a88 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/e0caae33f41e4a4cad87152d44f25a88 2024-11-16T10:26:17,349 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7636762808bf3c8f7a7fd5b283d2de9b/B of 7636762808bf3c8f7a7fd5b283d2de9b into e0caae33f41e4a4cad87152d44f25a88(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:26:17,349 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:17,349 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., storeName=7636762808bf3c8f7a7fd5b283d2de9b/B, priority=13, startTime=1731752776898; duration=0sec 2024-11-16T10:26:17,349 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:26:17,349 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7636762808bf3c8f7a7fd5b283d2de9b:B 2024-11-16T10:26:17,349 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:26:17,350 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:26:17,350 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): 7636762808bf3c8f7a7fd5b283d2de9b/C is initiating minor compaction (all files) 2024-11-16T10:26:17,350 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7636762808bf3c8f7a7fd5b283d2de9b/C in TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:17,350 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/afce63b7ac4a4ca3a3c9f14ff9d5d860, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/4da28a9a4b4f412e825992c357a3e369, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/cc941b3146434341a7279ad4a58674a5] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp, totalSize=35.3 K 2024-11-16T10:26:17,350 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting afce63b7ac4a4ca3a3c9f14ff9d5d860, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731752772511 2024-11-16T10:26:17,350 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 4da28a9a4b4f412e825992c357a3e369, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731752772651 2024-11-16T10:26:17,351 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting cc941b3146434341a7279ad4a58674a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731752774819 2024-11-16T10:26:17,362 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7636762808bf3c8f7a7fd5b283d2de9b#C#compaction#449 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:26:17,362 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/6571a467d21b46479f9dc0591c3b8b57 is 50, key is test_row_0/C:col10/1731752774819/Put/seqid=0 2024-11-16T10:26:17,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742348_1524 (size=12207) 2024-11-16T10:26:17,384 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7636762808bf3c8f7a7fd5b283d2de9b#A#compaction#447 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:26:17,385 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/e8f7f7b3e5f141deaec5878cbc4f166a is 175, key is test_row_0/A:col10/1731752774819/Put/seqid=0 2024-11-16T10:26:17,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742349_1525 (size=31161) 2024-11-16T10:26:17,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-16T10:26:17,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:17,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:17,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:17,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:17,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:17,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:17,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T10:26:17,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:17,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752837466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:17,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:17,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:17,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752837466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:17,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:17,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:17,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752837476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:17,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752837476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:17,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752837476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:17,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:17,540 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:26:17,551 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169a59dab5729b487b9422e562f02cb784_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169a59dab5729b487b9422e562f02cb784_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:17,552 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/e49ad4ebd83047eb8b1df6094d651e61, store: [table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:17,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/e49ad4ebd83047eb8b1df6094d651e61 is 175, key is test_row_0/A:col10/1731752777128/Put/seqid=0 2024-11-16T10:26:17,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742350_1526 (size=30955) 2024-11-16T10:26:17,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-16T10:26:17,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:17,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:17,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:17,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:17,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:17,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:17,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-16T10:26:17,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:17,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:17,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:17,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:17,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:17,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:17,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T10:26:17,773 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/6571a467d21b46479f9dc0591c3b8b57 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/6571a467d21b46479f9dc0591c3b8b57 2024-11-16T10:26:17,776 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7636762808bf3c8f7a7fd5b283d2de9b/C of 7636762808bf3c8f7a7fd5b283d2de9b into 6571a467d21b46479f9dc0591c3b8b57(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:26:17,776 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:17,776 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., storeName=7636762808bf3c8f7a7fd5b283d2de9b/C, priority=13, startTime=1731752776898; duration=0sec 2024-11-16T10:26:17,776 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:17,776 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7636762808bf3c8f7a7fd5b283d2de9b:C 2024-11-16T10:26:17,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:17,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752837777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:17,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:17,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752837777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:17,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:17,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:17,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:17,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752837786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:17,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:17,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752837786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:17,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:17,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:17,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752837786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:17,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:17,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:17,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:17,795 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/e8f7f7b3e5f141deaec5878cbc4f166a as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/e8f7f7b3e5f141deaec5878cbc4f166a 2024-11-16T10:26:17,798 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7636762808bf3c8f7a7fd5b283d2de9b/A of 7636762808bf3c8f7a7fd5b283d2de9b into e8f7f7b3e5f141deaec5878cbc4f166a(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:26:17,798 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:17,798 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., storeName=7636762808bf3c8f7a7fd5b283d2de9b/A, priority=13, startTime=1731752776898; duration=0sec 2024-11-16T10:26:17,799 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:17,799 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7636762808bf3c8f7a7fd5b283d2de9b:A 2024-11-16T10:26:17,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-16T10:26:17,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:17,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:17,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:17,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:17,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:17,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:17,960 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/e49ad4ebd83047eb8b1df6094d651e61 2024-11-16T10:26:17,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/328dcf3d02cb4400ace823b4283ff85c is 50, key is test_row_0/B:col10/1731752777128/Put/seqid=0 2024-11-16T10:26:17,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742351_1527 (size=12001) 2024-11-16T10:26:18,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-16T10:26:18,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:18,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:18,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:18,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:18,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:18,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-16T10:26:18,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:18,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:18,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:18,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:18,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T10:26:18,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:18,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:18,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752838286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:18,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752838286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:18,289 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:18,289 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:18,289 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:18,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:18,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:18,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:18,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752838296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:18,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:18,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:18,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:18,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752838296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:18,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:18,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:18,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:18,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752838296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:18,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:18,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:18,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:18,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-16T10:26:18,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:18,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:18,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:18,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:18,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:18,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/328dcf3d02cb4400ace823b4283ff85c 2024-11-16T10:26:18,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/b95a2c03ad4b4504b6b5523d805016fb is 50, key is test_row_0/C:col10/1731752777128/Put/seqid=0 2024-11-16T10:26:18,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742352_1528 (size=12001) 2024-11-16T10:26:18,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-16T10:26:18,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:18,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:18,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:18,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:18,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:18,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:18,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-16T10:26:18,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:18,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:18,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:18,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:18,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:18,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:18,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/b95a2c03ad4b4504b6b5523d805016fb 2024-11-16T10:26:18,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/e49ad4ebd83047eb8b1df6094d651e61 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/e49ad4ebd83047eb8b1df6094d651e61 2024-11-16T10:26:18,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/e49ad4ebd83047eb8b1df6094d651e61, entries=150, sequenceid=117, filesize=30.2 K 2024-11-16T10:26:18,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/328dcf3d02cb4400ace823b4283ff85c as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/328dcf3d02cb4400ace823b4283ff85c 2024-11-16T10:26:18,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/328dcf3d02cb4400ace823b4283ff85c, entries=150, sequenceid=117, filesize=11.7 K 2024-11-16T10:26:18,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/b95a2c03ad4b4504b6b5523d805016fb as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/b95a2c03ad4b4504b6b5523d805016fb 2024-11-16T10:26:18,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/b95a2c03ad4b4504b6b5523d805016fb, entries=150, sequenceid=117, filesize=11.7 K 2024-11-16T10:26:18,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 7636762808bf3c8f7a7fd5b283d2de9b in 1667ms, sequenceid=117, compaction requested=false 2024-11-16T10:26:18,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:18,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-16T10:26:18,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:18,809 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 7636762808bf3c8f7a7fd5b283d2de9b 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T10:26:18,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116aaac2f53221e4541a8fa741d39d2e1d4_7636762808bf3c8f7a7fd5b283d2de9b is 50, key is test_row_0/A:col10/1731752777149/Put/seqid=0 2024-11-16T10:26:18,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742353_1529 (size=12204) 2024-11-16T10:26:18,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:26:18,822 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116aaac2f53221e4541a8fa741d39d2e1d4_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116aaac2f53221e4541a8fa741d39d2e1d4_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:18,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/131c384a3e4345c99fc65084fca47f52, store: [table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:18,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/131c384a3e4345c99fc65084fca47f52 is 175, key is test_row_0/A:col10/1731752777149/Put/seqid=0 2024-11-16T10:26:18,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742354_1530 (size=31005) 2024-11-16T10:26:19,242 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/131c384a3e4345c99fc65084fca47f52 2024-11-16T10:26:19,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/e1e542a0367e41888121e5ba044cb52e is 50, key is test_row_0/B:col10/1731752777149/Put/seqid=0 2024-11-16T10:26:19,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742355_1531 (size=12051) 2024-11-16T10:26:19,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T10:26:19,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:19,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:19,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752839329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:19,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:19,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752839330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:19,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752839331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:19,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:19,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:19,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752839333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:19,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752839333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:19,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:19,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:19,338 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,338 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,338 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:19,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:19,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:19,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752839446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:19,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:19,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752839446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:19,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:19,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752839446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:19,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:19,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752839446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:19,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:19,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752839447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:19,650 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/e1e542a0367e41888121e5ba044cb52e 2024-11-16T10:26:19,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/1e662aeb69024345ae2f01fb60c02e62 is 50, key is test_row_0/C:col10/1731752777149/Put/seqid=0 2024-11-16T10:26:19,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742356_1532 (size=12051) 2024-11-16T10:26:19,659 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/1e662aeb69024345ae2f01fb60c02e62 2024-11-16T10:26:19,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:19,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752839657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:19,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:19,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752839657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:19,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:19,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752839657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:19,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:19,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752839657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:19,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:19,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752839657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:19,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:19,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:19,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:19,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:19,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:19,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/131c384a3e4345c99fc65084fca47f52 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/131c384a3e4345c99fc65084fca47f52 2024-11-16T10:26:19,666 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/131c384a3e4345c99fc65084fca47f52, entries=150, sequenceid=133, filesize=30.3 K 2024-11-16T10:26:19,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/e1e542a0367e41888121e5ba044cb52e as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/e1e542a0367e41888121e5ba044cb52e 2024-11-16T10:26:19,669 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/e1e542a0367e41888121e5ba044cb52e, entries=150, sequenceid=133, filesize=11.8 K 2024-11-16T10:26:19,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/1e662aeb69024345ae2f01fb60c02e62 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/1e662aeb69024345ae2f01fb60c02e62 2024-11-16T10:26:19,673 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/1e662aeb69024345ae2f01fb60c02e62, entries=150, sequenceid=133, filesize=11.8 K 2024-11-16T10:26:19,674 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 7636762808bf3c8f7a7fd5b283d2de9b in 865ms, sequenceid=133, compaction requested=true 2024-11-16T10:26:19,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:19,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:19,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-16T10:26:19,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-16T10:26:19,677 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-16T10:26:19,677 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5450 sec 2024-11-16T10:26:19,678 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.5500 sec 2024-11-16T10:26:19,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:19,968 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:26:19,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7636762808bf3c8f7a7fd5b283d2de9b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T10:26:19,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162fe23276ddf34663b0c0b94d98303472_7636762808bf3c8f7a7fd5b283d2de9b is 50, key is test_row_0/A:col10/1731752779967/Put/seqid=0 2024-11-16T10:26:19,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:19,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752839974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:19,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:19,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752839975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:19,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:19,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:19,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:19,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752839976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:19,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:19,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:19,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752839977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:19,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:19,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752839977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:19,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:19,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:19,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:19,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742357_1533 (size=12304) 2024-11-16T10:26:19,983 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:26:19,986 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162fe23276ddf34663b0c0b94d98303472_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162fe23276ddf34663b0c0b94d98303472_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:19,987 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/bd05e391afc943d695552f5c6adb6b42, store: [table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:19,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/bd05e391afc943d695552f5c6adb6b42 is 175, key is test_row_0/A:col10/1731752779967/Put/seqid=0 2024-11-16T10:26:19,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742358_1534 (size=31105) 2024-11-16T10:26:20,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:20,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752840086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:20,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752840086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:20,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752840086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:20,088 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:20,088 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,088 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:20,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752840086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:20,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:20,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:20,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752840086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:20,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:20,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:20,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752840296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:20,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:20,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752840296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:20,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:20,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752840296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:20,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:20,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:20,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752840296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:20,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752840296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:20,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:20,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:20,390 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/bd05e391afc943d695552f5c6adb6b42 2024-11-16T10:26:20,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/48e3dbf654cb43d892da301c1b5e2428 is 50, key is test_row_0/B:col10/1731752779967/Put/seqid=0 2024-11-16T10:26:20,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742359_1535 (size=12151) 2024-11-16T10:26:20,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:20,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752840606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:20,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:20,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752840606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:20,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752840606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:20,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:20,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752840606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:20,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:20,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752840616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:20,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:20,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:20,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/48e3dbf654cb43d892da301c1b5e2428 2024-11-16T10:26:20,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/ad729398ca0647e5b1f5ef5beb517528 is 50, key is test_row_0/C:col10/1731752779967/Put/seqid=0 2024-11-16T10:26:20,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742360_1536 (size=12151) 2024-11-16T10:26:21,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:21,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752841116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:21,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:21,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752841116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:21,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:21,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752841116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:21,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:21,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752841116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:21,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:21,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752841126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:21,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:21,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:21,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:21,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/ad729398ca0647e5b1f5ef5beb517528 2024-11-16T10:26:21,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/bd05e391afc943d695552f5c6adb6b42 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/bd05e391afc943d695552f5c6adb6b42 2024-11-16T10:26:21,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/bd05e391afc943d695552f5c6adb6b42, entries=150, sequenceid=158, filesize=30.4 K 2024-11-16T10:26:21,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/48e3dbf654cb43d892da301c1b5e2428 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/48e3dbf654cb43d892da301c1b5e2428 2024-11-16T10:26:21,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/48e3dbf654cb43d892da301c1b5e2428, entries=150, sequenceid=158, filesize=11.9 K 2024-11-16T10:26:21,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/ad729398ca0647e5b1f5ef5beb517528 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/ad729398ca0647e5b1f5ef5beb517528 2024-11-16T10:26:21,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/ad729398ca0647e5b1f5ef5beb517528, entries=150, sequenceid=158, filesize=11.9 K 2024-11-16T10:26:21,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 7636762808bf3c8f7a7fd5b283d2de9b in 1265ms, sequenceid=158, compaction requested=true 2024-11-16T10:26:21,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:21,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7636762808bf3c8f7a7fd5b283d2de9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:26:21,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:21,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7636762808bf3c8f7a7fd5b283d2de9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:26:21,233 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:26:21,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:21,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7636762808bf3c8f7a7fd5b283d2de9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:26:21,233 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:26:21,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:26:21,234 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:26:21,234 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124226 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:26:21,234 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): 7636762808bf3c8f7a7fd5b283d2de9b/A is initiating minor compaction (all files) 2024-11-16T10:26:21,234 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): 7636762808bf3c8f7a7fd5b283d2de9b/B is initiating minor compaction (all files) 2024-11-16T10:26:21,234 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7636762808bf3c8f7a7fd5b283d2de9b/B in TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:21,234 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/e0caae33f41e4a4cad87152d44f25a88, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/328dcf3d02cb4400ace823b4283ff85c, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/e1e542a0367e41888121e5ba044cb52e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/48e3dbf654cb43d892da301c1b5e2428] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp, totalSize=47.3 K 2024-11-16T10:26:21,234 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7636762808bf3c8f7a7fd5b283d2de9b/A in TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:21,235 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/e8f7f7b3e5f141deaec5878cbc4f166a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/e49ad4ebd83047eb8b1df6094d651e61, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/131c384a3e4345c99fc65084fca47f52, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/bd05e391afc943d695552f5c6adb6b42] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp, totalSize=121.3 K 2024-11-16T10:26:21,235 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:21,235 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. files: [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/e8f7f7b3e5f141deaec5878cbc4f166a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/e49ad4ebd83047eb8b1df6094d651e61, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/131c384a3e4345c99fc65084fca47f52, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/bd05e391afc943d695552f5c6adb6b42] 2024-11-16T10:26:21,235 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting e0caae33f41e4a4cad87152d44f25a88, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731752774819 2024-11-16T10:26:21,235 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8f7f7b3e5f141deaec5878cbc4f166a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731752774819 2024-11-16T10:26:21,236 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 328dcf3d02cb4400ace823b4283ff85c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731752775979 2024-11-16T10:26:21,236 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting e49ad4ebd83047eb8b1df6094d651e61, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731752775979 2024-11-16T10:26:21,236 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting e1e542a0367e41888121e5ba044cb52e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731752777146 2024-11-16T10:26:21,236 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 48e3dbf654cb43d892da301c1b5e2428, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731752779331 2024-11-16T10:26:21,236 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 131c384a3e4345c99fc65084fca47f52, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731752777146 2024-11-16T10:26:21,236 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd05e391afc943d695552f5c6adb6b42, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731752779331 2024-11-16T10:26:21,241 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:21,246 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7636762808bf3c8f7a7fd5b283d2de9b#B#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:26:21,247 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/464f2392a7f84c658d9280ed04eccb44 is 50, key is test_row_0/B:col10/1731752779967/Put/seqid=0 2024-11-16T10:26:21,250 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411169f2887d66e0945dfbd66d3d6e646ab58_7636762808bf3c8f7a7fd5b283d2de9b store=[table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:21,253 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411169f2887d66e0945dfbd66d3d6e646ab58_7636762808bf3c8f7a7fd5b283d2de9b, store=[table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:21,253 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169f2887d66e0945dfbd66d3d6e646ab58_7636762808bf3c8f7a7fd5b283d2de9b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:21,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742361_1537 (size=12493) 2024-11-16T10:26:21,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742362_1538 (size=4469) 2024-11-16T10:26:21,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T10:26:21,297 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:26:21,297 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:26:21,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-16T10:26:21,299 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:26:21,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T10:26:21,299 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:26:21,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:26:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T10:26:21,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T10:26:21,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:21,451 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 7636762808bf3c8f7a7fd5b283d2de9b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T10:26:21,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e8ae80f1bbae4570a732d56f349c3b19_7636762808bf3c8f7a7fd5b283d2de9b is 50, key is test_row_0/A:col10/1731752779976/Put/seqid=0 2024-11-16T10:26:21,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742363_1539 (size=12304) 2024-11-16T10:26:21,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T10:26:21,677 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/464f2392a7f84c658d9280ed04eccb44 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/464f2392a7f84c658d9280ed04eccb44 2024-11-16T10:26:21,681 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7636762808bf3c8f7a7fd5b283d2de9b/B of 7636762808bf3c8f7a7fd5b283d2de9b into 464f2392a7f84c658d9280ed04eccb44(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:26:21,681 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:21,681 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., storeName=7636762808bf3c8f7a7fd5b283d2de9b/B, priority=12, startTime=1731752781233; duration=0sec 2024-11-16T10:26:21,681 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:26:21,681 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7636762808bf3c8f7a7fd5b283d2de9b:B 2024-11-16T10:26:21,681 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:26:21,682 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:26:21,682 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): 7636762808bf3c8f7a7fd5b283d2de9b/C is initiating minor compaction (all files) 2024-11-16T10:26:21,682 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7636762808bf3c8f7a7fd5b283d2de9b/C in TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:21,682 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/6571a467d21b46479f9dc0591c3b8b57, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/b95a2c03ad4b4504b6b5523d805016fb, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/1e662aeb69024345ae2f01fb60c02e62, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/ad729398ca0647e5b1f5ef5beb517528] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp, totalSize=47.3 K 2024-11-16T10:26:21,682 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 6571a467d21b46479f9dc0591c3b8b57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731752774819 2024-11-16T10:26:21,683 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting b95a2c03ad4b4504b6b5523d805016fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731752775979 2024-11-16T10:26:21,683 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e662aeb69024345ae2f01fb60c02e62, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731752777146 2024-11-16T10:26:21,683 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting ad729398ca0647e5b1f5ef5beb517528, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731752779331 2024-11-16T10:26:21,686 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7636762808bf3c8f7a7fd5b283d2de9b#A#compaction#458 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:26:21,686 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/8c2ec46cf8c44d77b17a9d9d4dea5873 is 175, key is test_row_0/A:col10/1731752779967/Put/seqid=0 2024-11-16T10:26:21,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742364_1540 (size=31447) 2024-11-16T10:26:21,694 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7636762808bf3c8f7a7fd5b283d2de9b#C#compaction#461 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:26:21,694 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/646f47c8f3a7431b998a5de6a9ed3947 is 50, key is test_row_0/C:col10/1731752779967/Put/seqid=0 2024-11-16T10:26:21,698 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/8c2ec46cf8c44d77b17a9d9d4dea5873 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/8c2ec46cf8c44d77b17a9d9d4dea5873 2024-11-16T10:26:21,703 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7636762808bf3c8f7a7fd5b283d2de9b/A of 7636762808bf3c8f7a7fd5b283d2de9b into 8c2ec46cf8c44d77b17a9d9d4dea5873(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:26:21,703 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:21,703 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., storeName=7636762808bf3c8f7a7fd5b283d2de9b/A, priority=12, startTime=1731752781233; duration=0sec 2024-11-16T10:26:21,703 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:21,703 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7636762808bf3c8f7a7fd5b283d2de9b:A 2024-11-16T10:26:21,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742365_1541 (size=12493) 2024-11-16T10:26:21,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:26:21,876 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e8ae80f1bbae4570a732d56f349c3b19_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e8ae80f1bbae4570a732d56f349c3b19_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:21,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/5b77232498484f77b36399735dffb63a, store: [table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:21,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/5b77232498484f77b36399735dffb63a is 175, key is test_row_0/A:col10/1731752779976/Put/seqid=0 2024-11-16T10:26:21,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742366_1542 (size=31105) 2024-11-16T10:26:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T10:26:22,106 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/646f47c8f3a7431b998a5de6a9ed3947 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/646f47c8f3a7431b998a5de6a9ed3947 2024-11-16T10:26:22,109 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7636762808bf3c8f7a7fd5b283d2de9b/C of 7636762808bf3c8f7a7fd5b283d2de9b into 646f47c8f3a7431b998a5de6a9ed3947(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:26:22,109 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:22,109 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., storeName=7636762808bf3c8f7a7fd5b283d2de9b/C, priority=12, startTime=1731752781233; duration=0sec 2024-11-16T10:26:22,110 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:22,110 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7636762808bf3c8f7a7fd5b283d2de9b:C 2024-11-16T10:26:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:22,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:22,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752842215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:22,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752842217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:22,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:22,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752842217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:22,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:22,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:22,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:22,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752842222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752842222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:22,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:22,280 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/5b77232498484f77b36399735dffb63a 2024-11-16T10:26:22,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/9af784f4a5dc4629be788b806e3c4fd8 is 50, key is test_row_0/B:col10/1731752779976/Put/seqid=0 2024-11-16T10:26:22,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742367_1543 (size=12151) 2024-11-16T10:26:22,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752842326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752842326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752842326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:22,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752842336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752842336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T10:26:22,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752842536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:22,539 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:22,539 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,539 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752842536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:22,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:22,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752842536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:22,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:22,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752842546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:22,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:22,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752842546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:22,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:22,709 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/9af784f4a5dc4629be788b806e3c4fd8 2024-11-16T10:26:22,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/6e27d11518524447beb03468e5028a19 is 50, key is test_row_0/C:col10/1731752779976/Put/seqid=0 2024-11-16T10:26:22,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742368_1544 (size=12151) 2024-11-16T10:26:22,723 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/6e27d11518524447beb03468e5028a19 2024-11-16T10:26:22,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/5b77232498484f77b36399735dffb63a as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/5b77232498484f77b36399735dffb63a 2024-11-16T10:26:22,730 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/5b77232498484f77b36399735dffb63a, entries=150, sequenceid=170, filesize=30.4 K 2024-11-16T10:26:22,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/9af784f4a5dc4629be788b806e3c4fd8 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/9af784f4a5dc4629be788b806e3c4fd8 2024-11-16T10:26:22,734 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/9af784f4a5dc4629be788b806e3c4fd8, entries=150, sequenceid=170, filesize=11.9 K 2024-11-16T10:26:22,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/6e27d11518524447beb03468e5028a19 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/6e27d11518524447beb03468e5028a19 2024-11-16T10:26:22,738 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/6e27d11518524447beb03468e5028a19, entries=150, sequenceid=170, filesize=11.9 K 2024-11-16T10:26:22,739 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 7636762808bf3c8f7a7fd5b283d2de9b in 1287ms, sequenceid=170, compaction requested=false 2024-11-16T10:26:22,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:22,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:22,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-16T10:26:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-16T10:26:22,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-16T10:26:22,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4410 sec 2024-11-16T10:26:22,746 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.4480 sec 2024-11-16T10:26:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:22,848 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:26:22,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7636762808bf3c8f7a7fd5b283d2de9b 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-16T10:26:22,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162557d8d5cdd745b9a9f50dc518316293_7636762808bf3c8f7a7fd5b283d2de9b is 50, key is test_row_0/A:col10/1731752782848/Put/seqid=0 2024-11-16T10:26:22,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752842856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:22,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:22,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752842856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:22,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:22,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:22,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752842858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:22,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:22,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752842858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:22,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:22,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752842859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:22,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:22,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:22,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742369_1545 (size=17284) 2024-11-16T10:26:22,871 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:26:22,874 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162557d8d5cdd745b9a9f50dc518316293_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162557d8d5cdd745b9a9f50dc518316293_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:22,874 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/fe8fbb0da7d94521a093d2a9fc1083b7, store: [table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:22,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/fe8fbb0da7d94521a093d2a9fc1083b7 is 175, key is test_row_0/A:col10/1731752782848/Put/seqid=0 2024-11-16T10:26:22,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742370_1546 (size=48389) 2024-11-16T10:26:22,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752842966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:22,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:22,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752842966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:22,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:22,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752842966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:22,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:22,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:23,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752843176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:23,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:23,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752843176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:23,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:23,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752843176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:23,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:23,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:23,278 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=200, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/fe8fbb0da7d94521a093d2a9fc1083b7 2024-11-16T10:26:23,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/387f2f01aa1c4f2ca963382a02a583c4 is 50, key is test_row_0/B:col10/1731752782848/Put/seqid=0 2024-11-16T10:26:23,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742371_1547 (size=12151) 2024-11-16T10:26:23,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/387f2f01aa1c4f2ca963382a02a583c4 2024-11-16T10:26:23,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/4c7f04c7154e4ce899712f180d76414e is 50, key is test_row_0/C:col10/1731752782848/Put/seqid=0 2024-11-16T10:26:23,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742372_1548 (size=12151) 2024-11-16T10:26:23,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/4c7f04c7154e4ce899712f180d76414e 2024-11-16T10:26:23,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/fe8fbb0da7d94521a093d2a9fc1083b7 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/fe8fbb0da7d94521a093d2a9fc1083b7 2024-11-16T10:26:23,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/fe8fbb0da7d94521a093d2a9fc1083b7, entries=250, sequenceid=200, filesize=47.3 K 2024-11-16T10:26:23,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/387f2f01aa1c4f2ca963382a02a583c4 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/387f2f01aa1c4f2ca963382a02a583c4 2024-11-16T10:26:23,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/387f2f01aa1c4f2ca963382a02a583c4, entries=150, sequenceid=200, filesize=11.9 K 2024-11-16T10:26:23,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/4c7f04c7154e4ce899712f180d76414e as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/4c7f04c7154e4ce899712f180d76414e 2024-11-16T10:26:23,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/4c7f04c7154e4ce899712f180d76414e, entries=150, sequenceid=200, filesize=11.9 K 2024-11-16T10:26:23,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 7636762808bf3c8f7a7fd5b283d2de9b in 509ms, sequenceid=200, compaction requested=true 2024-11-16T10:26:23,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:23,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7636762808bf3c8f7a7fd5b283d2de9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:26:23,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:23,357 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:26:23,358 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:26:23,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7636762808bf3c8f7a7fd5b283d2de9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:26:23,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:23,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7636762808bf3c8f7a7fd5b283d2de9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:26:23,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:26:23,360 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110941 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:26:23,360 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:26:23,360 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): 7636762808bf3c8f7a7fd5b283d2de9b/A is initiating minor compaction (all files) 2024-11-16T10:26:23,360 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): 7636762808bf3c8f7a7fd5b283d2de9b/B is initiating minor compaction (all files) 2024-11-16T10:26:23,360 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7636762808bf3c8f7a7fd5b283d2de9b/A in TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:23,360 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7636762808bf3c8f7a7fd5b283d2de9b/B in TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:23,360 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/464f2392a7f84c658d9280ed04eccb44, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/9af784f4a5dc4629be788b806e3c4fd8, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/387f2f01aa1c4f2ca963382a02a583c4] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp, totalSize=35.9 K 2024-11-16T10:26:23,360 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/8c2ec46cf8c44d77b17a9d9d4dea5873, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/5b77232498484f77b36399735dffb63a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/fe8fbb0da7d94521a093d2a9fc1083b7] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp, totalSize=108.3 K 2024-11-16T10:26:23,360 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:23,361 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. files: [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/8c2ec46cf8c44d77b17a9d9d4dea5873, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/5b77232498484f77b36399735dffb63a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/fe8fbb0da7d94521a093d2a9fc1083b7] 2024-11-16T10:26:23,364 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 464f2392a7f84c658d9280ed04eccb44, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731752779331 2024-11-16T10:26:23,364 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c2ec46cf8c44d77b17a9d9d4dea5873, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731752779331 2024-11-16T10:26:23,365 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b77232498484f77b36399735dffb63a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731752779974 2024-11-16T10:26:23,365 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 9af784f4a5dc4629be788b806e3c4fd8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731752779974 2024-11-16T10:26:23,365 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe8fbb0da7d94521a093d2a9fc1083b7, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731752782215 2024-11-16T10:26:23,365 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 387f2f01aa1c4f2ca963382a02a583c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731752782221 2024-11-16T10:26:23,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:23,370 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:26:23,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7636762808bf3c8f7a7fd5b283d2de9b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T10:26:23,372 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:23,383 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7636762808bf3c8f7a7fd5b283d2de9b#B#compaction#468 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:26:23,383 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/fe342fe4975b4f71a35917f07469ed5e is 50, key is test_row_0/B:col10/1731752782848/Put/seqid=0 2024-11-16T10:26:23,389 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116874dc423048d400481fd3a2479ed6e39_7636762808bf3c8f7a7fd5b283d2de9b store=[table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:23,390 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116874dc423048d400481fd3a2479ed6e39_7636762808bf3c8f7a7fd5b283d2de9b, store=[table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:23,390 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116874dc423048d400481fd3a2479ed6e39_7636762808bf3c8f7a7fd5b283d2de9b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:23,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fa7f0d62b1714f9f94dc8aced44c576a_7636762808bf3c8f7a7fd5b283d2de9b is 50, key is test_row_0/A:col10/1731752783368/Put/seqid=0 2024-11-16T10:26:23,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742374_1550 (size=4469) 2024-11-16T10:26:23,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742375_1551 (size=14794) 2024-11-16T10:26:23,420 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:26:23,424 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fa7f0d62b1714f9f94dc8aced44c576a_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fa7f0d62b1714f9f94dc8aced44c576a_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:23,426 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/ab88024ed76f4d0685333f10157b3f18, store: [table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:23,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/ab88024ed76f4d0685333f10157b3f18 is 175, key is test_row_0/A:col10/1731752783368/Put/seqid=0 2024-11-16T10:26:23,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:23,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752843431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:23,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:23,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742373_1549 (size=12595) 2024-11-16T10:26:23,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:23,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752843433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:23,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:23,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742376_1552 (size=39749) 2024-11-16T10:26:23,442 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/ab88024ed76f4d0685333f10157b3f18 2024-11-16T10:26:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T10:26:23,447 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:26:23,447 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:26:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-16T10:26:23,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/d4dcd984ae0d40c593763c04f9c1209c is 50, key is test_row_0/B:col10/1731752783368/Put/seqid=0 2024-11-16T10:26:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T10:26:23,450 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:26:23,451 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:26:23,451 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:26:23,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742377_1553 (size=12151) 2024-11-16T10:26:23,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/d4dcd984ae0d40c593763c04f9c1209c 2024-11-16T10:26:23,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/9e02723e87e8456293b0a6d9cc3bb1fc is 50, key is test_row_0/C:col10/1731752783368/Put/seqid=0 2024-11-16T10:26:23,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742378_1554 (size=12151) 2024-11-16T10:26:23,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/9e02723e87e8456293b0a6d9cc3bb1fc 2024-11-16T10:26:23,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/ab88024ed76f4d0685333f10157b3f18 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/ab88024ed76f4d0685333f10157b3f18 2024-11-16T10:26:23,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/ab88024ed76f4d0685333f10157b3f18, entries=200, sequenceid=212, filesize=38.8 K 2024-11-16T10:26:23,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/d4dcd984ae0d40c593763c04f9c1209c as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/d4dcd984ae0d40c593763c04f9c1209c 2024-11-16T10:26:23,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/d4dcd984ae0d40c593763c04f9c1209c, entries=150, sequenceid=212, filesize=11.9 K 2024-11-16T10:26:23,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/9e02723e87e8456293b0a6d9cc3bb1fc as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/9e02723e87e8456293b0a6d9cc3bb1fc 2024-11-16T10:26:23,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/9e02723e87e8456293b0a6d9cc3bb1fc, entries=150, sequenceid=212, filesize=11.9 K 2024-11-16T10:26:23,479 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 7636762808bf3c8f7a7fd5b283d2de9b in 109ms, sequenceid=212, compaction requested=true 2024-11-16T10:26:23,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:23,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7636762808bf3c8f7a7fd5b283d2de9b:A, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:26:23,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T10:26:23,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7636762808bf3c8f7a7fd5b283d2de9b:B, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:26:23,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-16T10:26:23,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7636762808bf3c8f7a7fd5b283d2de9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:26:23,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-16T10:26:23,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:23,488 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:26:23,488 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7636762808bf3c8f7a7fd5b283d2de9b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T10:26:23,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160385df09e9b34edb98151ffa7e4e1b17_7636762808bf3c8f7a7fd5b283d2de9b is 50, key is test_row_0/A:col10/1731752783429/Put/seqid=0 2024-11-16T10:26:23,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752843502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752843503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:23,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752843504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:23,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:23,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742379_1555 (size=14794) 2024-11-16T10:26:23,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752843546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:23,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:23,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752843546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:23,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:23,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T10:26:23,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T10:26:23,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:23,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:23,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:23,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:23,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:23,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:23,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:23,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752843616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:23,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:23,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752843616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:23,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752843616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:23,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T10:26:23,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:23,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:23,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:23,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:23,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:23,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:23,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752843756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:23,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:23,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752843756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:23,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T10:26:23,820 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7636762808bf3c8f7a7fd5b283d2de9b#A#compaction#467 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:26:23,821 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/f17a2675bd52452c9d5dd43d12fe445f is 175, key is test_row_0/A:col10/1731752782848/Put/seqid=0 2024-11-16T10:26:23,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742380_1556 (size=31549) 2024-11-16T10:26:23,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752843826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:23,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752843827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:23,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:23,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:23,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752843827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:23,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:23,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:23,841 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/fe342fe4975b4f71a35917f07469ed5e as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/fe342fe4975b4f71a35917f07469ed5e 2024-11-16T10:26:23,844 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7636762808bf3c8f7a7fd5b283d2de9b/B of 7636762808bf3c8f7a7fd5b283d2de9b into fe342fe4975b4f71a35917f07469ed5e(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:26:23,844 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:23,844 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., storeName=7636762808bf3c8f7a7fd5b283d2de9b/B, priority=13, startTime=1731752783358; duration=0sec 2024-11-16T10:26:23,844 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-16T10:26:23,844 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7636762808bf3c8f7a7fd5b283d2de9b:B 2024-11-16T10:26:23,844 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-16T10:26:23,844 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-16T10:26:23,845 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-16T10:26:23,845 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. because compaction request was cancelled 2024-11-16T10:26:23,845 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7636762808bf3c8f7a7fd5b283d2de9b:A 2024-11-16T10:26:23,845 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:26:23,846 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:26:23,846 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): 7636762808bf3c8f7a7fd5b283d2de9b/C is initiating minor compaction (all files) 2024-11-16T10:26:23,846 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7636762808bf3c8f7a7fd5b283d2de9b/C in TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:23,846 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/646f47c8f3a7431b998a5de6a9ed3947, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/6e27d11518524447beb03468e5028a19, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/4c7f04c7154e4ce899712f180d76414e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/9e02723e87e8456293b0a6d9cc3bb1fc] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp, totalSize=47.8 K 2024-11-16T10:26:23,846 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 646f47c8f3a7431b998a5de6a9ed3947, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731752779331 2024-11-16T10:26:23,846 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e27d11518524447beb03468e5028a19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731752779974 2024-11-16T10:26:23,846 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c7f04c7154e4ce899712f180d76414e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731752782221 2024-11-16T10:26:23,847 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e02723e87e8456293b0a6d9cc3bb1fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731752782857 2024-11-16T10:26:23,853 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7636762808bf3c8f7a7fd5b283d2de9b#C#compaction#473 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:26:23,854 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/a35d900ddb404b74a4a809dc54f055bc is 50, key is test_row_0/C:col10/1731752783368/Put/seqid=0 2024-11-16T10:26:23,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742381_1557 (size=12629) 2024-11-16T10:26:23,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T10:26:23,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:23,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:23,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:23,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:23,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:23,909 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:26:23,912 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160385df09e9b34edb98151ffa7e4e1b17_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160385df09e9b34edb98151ffa7e4e1b17_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:23,912 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/4a23fe391d414defa995c65d3f2ad3ff, store: [table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:23,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/4a23fe391d414defa995c65d3f2ad3ff is 175, key is test_row_0/A:col10/1731752783429/Put/seqid=0 2024-11-16T10:26:23,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742382_1558 (size=39749) 2024-11-16T10:26:24,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T10:26:24,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:24,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:24,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:24,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:24,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:24,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752844066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:24,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752844066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T10:26:24,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:24,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752844136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:24,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:24,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:24,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:24,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:24,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752844136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:24,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:24,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752844136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:24,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T10:26:24,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:24,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:24,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:24,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:24,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:24,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:24,228 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/f17a2675bd52452c9d5dd43d12fe445f as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/f17a2675bd52452c9d5dd43d12fe445f 2024-11-16T10:26:24,232 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7636762808bf3c8f7a7fd5b283d2de9b/A of 7636762808bf3c8f7a7fd5b283d2de9b into f17a2675bd52452c9d5dd43d12fe445f(size=30.8 K), total size for store is 69.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:26:24,232 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:24,232 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., storeName=7636762808bf3c8f7a7fd5b283d2de9b/A, priority=13, startTime=1731752783357; duration=0sec 2024-11-16T10:26:24,232 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T10:26:24,232 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7636762808bf3c8f7a7fd5b283d2de9b:A 2024-11-16T10:26:24,232 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7636762808bf3c8f7a7fd5b283d2de9b:B 2024-11-16T10:26:24,232 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-16T10:26:24,232 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-16T10:26:24,232 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-16T10:26:24,233 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. because compaction request was cancelled 2024-11-16T10:26:24,233 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7636762808bf3c8f7a7fd5b283d2de9b:C 2024-11-16T10:26:24,233 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-16T10:26:24,233 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-16T10:26:24,234 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-16T10:26:24,234 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. because compaction request was cancelled 2024-11-16T10:26:24,234 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7636762808bf3c8f7a7fd5b283d2de9b:B 2024-11-16T10:26:24,269 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/a35d900ddb404b74a4a809dc54f055bc as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/a35d900ddb404b74a4a809dc54f055bc 2024-11-16T10:26:24,272 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7636762808bf3c8f7a7fd5b283d2de9b/C of 7636762808bf3c8f7a7fd5b283d2de9b into a35d900ddb404b74a4a809dc54f055bc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:26:24,272 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:24,272 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., storeName=7636762808bf3c8f7a7fd5b283d2de9b/C, priority=12, startTime=1731752783479; duration=0sec 2024-11-16T10:26:24,272 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:24,272 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7636762808bf3c8f7a7fd5b283d2de9b:C 2024-11-16T10:26:24,316 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/4a23fe391d414defa995c65d3f2ad3ff 2024-11-16T10:26:24,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/5cbedee5a89a487f9a8078883595660a is 50, key is test_row_0/B:col10/1731752783429/Put/seqid=0 2024-11-16T10:26:24,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742383_1559 (size=12151) 2024-11-16T10:26:24,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/5cbedee5a89a487f9a8078883595660a 2024-11-16T10:26:24,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/72181371e60b4d158fc347ddf58068fd is 50, key is test_row_0/C:col10/1731752783429/Put/seqid=0 2024-11-16T10:26:24,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742384_1560 (size=12151) 2024-11-16T10:26:24,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T10:26:24,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:24,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:24,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:24,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:24,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:24,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T10:26:24,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:24,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:24,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:24,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:24,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:24,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:24,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752844576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:24,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:24,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:24,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:24,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752844576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:24,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T10:26:24,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:24,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752844646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:24,650 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:24,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:24,650 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:24,650 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752844646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:24,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:24,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:24,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:24,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752844646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:24,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:24,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:24,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:24,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T10:26:24,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:24,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:24,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:24,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:24,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:24,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:24,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/72181371e60b4d158fc347ddf58068fd 2024-11-16T10:26:24,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/4a23fe391d414defa995c65d3f2ad3ff as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/4a23fe391d414defa995c65d3f2ad3ff 2024-11-16T10:26:24,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/4a23fe391d414defa995c65d3f2ad3ff, entries=200, sequenceid=237, filesize=38.8 K 2024-11-16T10:26:24,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/5cbedee5a89a487f9a8078883595660a as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/5cbedee5a89a487f9a8078883595660a 2024-11-16T10:26:24,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/5cbedee5a89a487f9a8078883595660a, entries=150, sequenceid=237, filesize=11.9 K 2024-11-16T10:26:24,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/72181371e60b4d158fc347ddf58068fd as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/72181371e60b4d158fc347ddf58068fd 2024-11-16T10:26:24,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/72181371e60b4d158fc347ddf58068fd, entries=150, sequenceid=237, filesize=11.9 K 2024-11-16T10:26:24,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 7636762808bf3c8f7a7fd5b283d2de9b in 1266ms, sequenceid=237, compaction requested=true 2024-11-16T10:26:24,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:24,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7636762808bf3c8f7a7fd5b283d2de9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:26:24,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:24,755 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:26:24,755 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:26:24,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7636762808bf3c8f7a7fd5b283d2de9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:26:24,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:24,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7636762808bf3c8f7a7fd5b283d2de9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:26:24,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:26:24,759 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:26:24,759 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:26:24,759 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): 7636762808bf3c8f7a7fd5b283d2de9b/B is initiating minor compaction (all files) 2024-11-16T10:26:24,759 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): 7636762808bf3c8f7a7fd5b283d2de9b/A is initiating minor compaction (all files) 2024-11-16T10:26:24,759 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7636762808bf3c8f7a7fd5b283d2de9b/B in TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:24,759 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7636762808bf3c8f7a7fd5b283d2de9b/A in TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:24,759 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/fe342fe4975b4f71a35917f07469ed5e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/d4dcd984ae0d40c593763c04f9c1209c, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/5cbedee5a89a487f9a8078883595660a] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp, totalSize=36.0 K 2024-11-16T10:26:24,759 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/f17a2675bd52452c9d5dd43d12fe445f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/ab88024ed76f4d0685333f10157b3f18, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/4a23fe391d414defa995c65d3f2ad3ff] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp, totalSize=108.4 K 2024-11-16T10:26:24,759 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:24,759 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. files: [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/f17a2675bd52452c9d5dd43d12fe445f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/ab88024ed76f4d0685333f10157b3f18, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/4a23fe391d414defa995c65d3f2ad3ff] 2024-11-16T10:26:24,759 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting fe342fe4975b4f71a35917f07469ed5e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731752782221 2024-11-16T10:26:24,759 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting f17a2675bd52452c9d5dd43d12fe445f, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731752782221 2024-11-16T10:26:24,760 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting d4dcd984ae0d40c593763c04f9c1209c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731752782857 2024-11-16T10:26:24,760 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab88024ed76f4d0685333f10157b3f18, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731752782857 2024-11-16T10:26:24,760 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cbedee5a89a487f9a8078883595660a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731752783425 2024-11-16T10:26:24,760 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a23fe391d414defa995c65d3f2ad3ff, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731752783425 2024-11-16T10:26:24,767 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7636762808bf3c8f7a7fd5b283d2de9b#B#compaction#476 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:26:24,768 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/83c5e76f883244ac958035bcef2b7003 is 50, key is test_row_0/B:col10/1731752783429/Put/seqid=0 2024-11-16T10:26:24,773 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:24,777 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116c097ddf4171e4869aaa22a27c84612ad_7636762808bf3c8f7a7fd5b283d2de9b store=[table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:24,779 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116c097ddf4171e4869aaa22a27c84612ad_7636762808bf3c8f7a7fd5b283d2de9b, store=[table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:24,779 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c097ddf4171e4869aaa22a27c84612ad_7636762808bf3c8f7a7fd5b283d2de9b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:24,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742385_1561 (size=12697) 2024-11-16T10:26:24,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742386_1562 (size=4469) 2024-11-16T10:26:24,797 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7636762808bf3c8f7a7fd5b283d2de9b#A#compaction#477 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:26:24,798 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/d4b8ee34a749495bbf1cdcf59b91c484 is 175, key is test_row_0/A:col10/1731752783429/Put/seqid=0 2024-11-16T10:26:24,801 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/83c5e76f883244ac958035bcef2b7003 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/83c5e76f883244ac958035bcef2b7003 2024-11-16T10:26:24,806 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7636762808bf3c8f7a7fd5b283d2de9b/B of 7636762808bf3c8f7a7fd5b283d2de9b into 83c5e76f883244ac958035bcef2b7003(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:26:24,806 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:24,806 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., storeName=7636762808bf3c8f7a7fd5b283d2de9b/B, priority=13, startTime=1731752784755; duration=0sec 2024-11-16T10:26:24,807 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:26:24,807 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7636762808bf3c8f7a7fd5b283d2de9b:B 2024-11-16T10:26:24,807 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-16T10:26:24,808 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-16T10:26:24,808 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-16T10:26:24,808 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. because compaction request was cancelled 2024-11-16T10:26:24,808 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7636762808bf3c8f7a7fd5b283d2de9b:C 2024-11-16T10:26:24,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T10:26:24,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:24,823 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 7636762808bf3c8f7a7fd5b283d2de9b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T10:26:24,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742387_1563 (size=31651) 2024-11-16T10:26:24,828 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/d4b8ee34a749495bbf1cdcf59b91c484 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/d4b8ee34a749495bbf1cdcf59b91c484 2024-11-16T10:26:24,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116174c08df8adc4e698c18902d8270a7de_7636762808bf3c8f7a7fd5b283d2de9b is 50, key is test_row_0/A:col10/1731752783503/Put/seqid=0 2024-11-16T10:26:24,832 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7636762808bf3c8f7a7fd5b283d2de9b/A of 7636762808bf3c8f7a7fd5b283d2de9b into d4b8ee34a749495bbf1cdcf59b91c484(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:26:24,832 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:24,833 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., storeName=7636762808bf3c8f7a7fd5b283d2de9b/A, priority=13, startTime=1731752784755; duration=0sec 2024-11-16T10:26:24,833 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:24,833 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7636762808bf3c8f7a7fd5b283d2de9b:A 2024-11-16T10:26:24,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742388_1564 (size=12304) 2024-11-16T10:26:25,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:26:25,239 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116174c08df8adc4e698c18902d8270a7de_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116174c08df8adc4e698c18902d8270a7de_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:25,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/91543c4049fe4f24a0a062aecf36ea7f, store: [table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:25,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/91543c4049fe4f24a0a062aecf36ea7f is 175, key is test_row_0/A:col10/1731752783503/Put/seqid=0 2024-11-16T10:26:25,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742389_1565 (size=31105) 2024-11-16T10:26:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:25,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:25,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T10:26:25,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:25,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752845630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:25,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:25,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:25,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:25,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752845631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:25,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:25,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:25,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:25,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:25,646 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/91543c4049fe4f24a0a062aecf36ea7f 2024-11-16T10:26:25,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/ff526256c4f5429081bdc51012c8c6e5 is 50, key is test_row_0/B:col10/1731752783503/Put/seqid=0 2024-11-16T10:26:25,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742390_1566 (size=12151) 2024-11-16T10:26:25,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:25,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752845656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:25,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:25,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752845656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:25,660 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_2' in 7636762808bf3c8f7a7fd5b283d2de9b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:26:25,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:25,660 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:25,660 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:25,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752845656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:25,660 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:25,660 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_1' in 7636762808bf3c8f7a7fd5b283d2de9b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:26:25,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:25,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:25,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:25,660 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_0' in 7636762808bf3c8f7a7fd5b283d2de9b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:26:25,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:25,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:25,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:25,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:25,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752845736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:25,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:25,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:25,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:25,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752845746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:25,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:25,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752845946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:25,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:25,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:25,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:25,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:25,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752845956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:25,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:25,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:25,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:26,055 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/ff526256c4f5429081bdc51012c8c6e5 2024-11-16T10:26:26,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/538f245882584e1c937764c6f4966695 is 50, key is test_row_0/C:col10/1731752783503/Put/seqid=0 2024-11-16T10:26:26,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742391_1567 (size=12151) 2024-11-16T10:26:26,064 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/538f245882584e1c937764c6f4966695 2024-11-16T10:26:26,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/91543c4049fe4f24a0a062aecf36ea7f as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/91543c4049fe4f24a0a062aecf36ea7f 2024-11-16T10:26:26,072 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/91543c4049fe4f24a0a062aecf36ea7f, entries=150, sequenceid=252, filesize=30.4 K 2024-11-16T10:26:26,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/ff526256c4f5429081bdc51012c8c6e5 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/ff526256c4f5429081bdc51012c8c6e5 2024-11-16T10:26:26,075 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/ff526256c4f5429081bdc51012c8c6e5, entries=150, sequenceid=252, filesize=11.9 K 2024-11-16T10:26:26,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/538f245882584e1c937764c6f4966695 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/538f245882584e1c937764c6f4966695 2024-11-16T10:26:26,079 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/538f245882584e1c937764c6f4966695, entries=150, sequenceid=252, filesize=11.9 K 2024-11-16T10:26:26,080 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 7636762808bf3c8f7a7fd5b283d2de9b in 1257ms, sequenceid=252, compaction requested=true 2024-11-16T10:26:26,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:26,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:26,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-16T10:26:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-16T10:26:26,083 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-16T10:26:26,083 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6300 sec 2024-11-16T10:26:26,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.6360 sec 2024-11-16T10:26:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:26,257 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:26:26,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7636762808bf3c8f7a7fd5b283d2de9b 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-16T10:26:26,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111699348bd3ab354c598c00ee3133d0b588_7636762808bf3c8f7a7fd5b283d2de9b is 50, key is test_row_0/A:col10/1731752786256/Put/seqid=0 2024-11-16T10:26:26,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742392_1568 (size=14994) 2024-11-16T10:26:26,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752846274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:26,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:26,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:26,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:26,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752846278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:26,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:26,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:26,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:26,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752846386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:26,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752846387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:26,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:26,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752846596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752846596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:26,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:26,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:26,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:26,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:26,667 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:26:26,670 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111699348bd3ab354c598c00ee3133d0b588_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111699348bd3ab354c598c00ee3133d0b588_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:26,670 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/29643a8b50e24b4897c997c7e8c13598, store: [table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:26,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/29643a8b50e24b4897c997c7e8c13598 is 175, key is test_row_0/A:col10/1731752786256/Put/seqid=0 2024-11-16T10:26:26,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742393_1569 (size=39949) 2024-11-16T10:26:26,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752846906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:26,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:26,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752846906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:26,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:26,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:26,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:26,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:26,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:27,078 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/29643a8b50e24b4897c997c7e8c13598 2024-11-16T10:26:27,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/ba9769c2bf0c4e078ecfba87baa9444d is 50, key is test_row_0/B:col10/1731752786256/Put/seqid=0 2024-11-16T10:26:27,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742394_1570 (size=12301) 2024-11-16T10:26:27,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:27,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752847416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:27,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:27,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:27,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:27,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:27,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752847416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:27,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:27,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:27,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:27,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/ba9769c2bf0c4e078ecfba87baa9444d 2024-11-16T10:26:27,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/e530a53e0aa7480880be996bb54b574f is 50, key is test_row_0/C:col10/1731752786256/Put/seqid=0 2024-11-16T10:26:27,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742395_1571 (size=12301) 2024-11-16T10:26:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T10:26:27,630 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:26:27,630 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:26:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-16T10:26:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T10:26:27,632 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:26:27,633 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:26:27,633 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:26:27,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752847666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:27,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752847666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:27,668 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_2' in 7636762808bf3c8f7a7fd5b283d2de9b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:26:27,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:27,668 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_0' in 7636762808bf3c8f7a7fd5b283d2de9b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:26:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:27,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:27,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752847677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:27,679 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_1' in 7636762808bf3c8f7a7fd5b283d2de9b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:26:27,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:27,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:27,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T10:26:27,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T10:26:27,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:27,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:27,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:27,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:27,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:27,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:27,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/e530a53e0aa7480880be996bb54b574f 2024-11-16T10:26:27,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/29643a8b50e24b4897c997c7e8c13598 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/29643a8b50e24b4897c997c7e8c13598 2024-11-16T10:26:27,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/29643a8b50e24b4897c997c7e8c13598, entries=200, sequenceid=279, filesize=39.0 K 2024-11-16T10:26:27,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/ba9769c2bf0c4e078ecfba87baa9444d as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/ba9769c2bf0c4e078ecfba87baa9444d 2024-11-16T10:26:27,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/ba9769c2bf0c4e078ecfba87baa9444d, entries=150, sequenceid=279, filesize=12.0 K 2024-11-16T10:26:27,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/e530a53e0aa7480880be996bb54b574f as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/e530a53e0aa7480880be996bb54b574f 2024-11-16T10:26:27,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/e530a53e0aa7480880be996bb54b574f, entries=150, sequenceid=279, filesize=12.0 K 2024-11-16T10:26:27,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 7636762808bf3c8f7a7fd5b283d2de9b in 1654ms, sequenceid=279, compaction requested=true 2024-11-16T10:26:27,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:27,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7636762808bf3c8f7a7fd5b283d2de9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:26:27,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:27,912 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:26:27,912 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:26:27,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7636762808bf3c8f7a7fd5b283d2de9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:26:27,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:27,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7636762808bf3c8f7a7fd5b283d2de9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:26:27,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:26:27,913 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:26:27,913 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:26:27,913 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): 7636762808bf3c8f7a7fd5b283d2de9b/A is initiating minor compaction (all files) 2024-11-16T10:26:27,913 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): 7636762808bf3c8f7a7fd5b283d2de9b/B is initiating minor compaction (all files) 2024-11-16T10:26:27,913 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7636762808bf3c8f7a7fd5b283d2de9b/A in TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:27,913 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7636762808bf3c8f7a7fd5b283d2de9b/B in TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:27,913 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/83c5e76f883244ac958035bcef2b7003, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/ff526256c4f5429081bdc51012c8c6e5, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/ba9769c2bf0c4e078ecfba87baa9444d] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp, totalSize=36.3 K 2024-11-16T10:26:27,913 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/d4b8ee34a749495bbf1cdcf59b91c484, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/91543c4049fe4f24a0a062aecf36ea7f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/29643a8b50e24b4897c997c7e8c13598] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp, totalSize=100.3 K 2024-11-16T10:26:27,913 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:27,913 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. files: [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/d4b8ee34a749495bbf1cdcf59b91c484, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/91543c4049fe4f24a0a062aecf36ea7f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/29643a8b50e24b4897c997c7e8c13598] 2024-11-16T10:26:27,913 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 83c5e76f883244ac958035bcef2b7003, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731752783425 2024-11-16T10:26:27,913 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4b8ee34a749495bbf1cdcf59b91c484, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731752783425 2024-11-16T10:26:27,914 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting ff526256c4f5429081bdc51012c8c6e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731752783500 2024-11-16T10:26:27,914 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91543c4049fe4f24a0a062aecf36ea7f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731752783500 2024-11-16T10:26:27,914 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting ba9769c2bf0c4e078ecfba87baa9444d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731752785625 2024-11-16T10:26:27,914 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29643a8b50e24b4897c997c7e8c13598, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731752785625 2024-11-16T10:26:27,921 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7636762808bf3c8f7a7fd5b283d2de9b#B#compaction#484 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:26:27,921 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/54205d9dc9724cb6a80f358332b98251 is 50, key is test_row_0/B:col10/1731752786256/Put/seqid=0 2024-11-16T10:26:27,922 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:27,933 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411161f3c3204a71142ab88b692e0e96b36f7_7636762808bf3c8f7a7fd5b283d2de9b store=[table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:27,935 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411161f3c3204a71142ab88b692e0e96b36f7_7636762808bf3c8f7a7fd5b283d2de9b, store=[table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:27,936 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161f3c3204a71142ab88b692e0e96b36f7_7636762808bf3c8f7a7fd5b283d2de9b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:27,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T10:26:27,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:27,939 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 7636762808bf3c8f7a7fd5b283d2de9b 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-16T10:26:27,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742396_1572 (size=12949) 2024-11-16T10:26:27,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T10:26:27,949 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/54205d9dc9724cb6a80f358332b98251 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/54205d9dc9724cb6a80f358332b98251 2024-11-16T10:26:27,952 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7636762808bf3c8f7a7fd5b283d2de9b/B of 7636762808bf3c8f7a7fd5b283d2de9b into 54205d9dc9724cb6a80f358332b98251(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:26:27,952 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:27,952 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., storeName=7636762808bf3c8f7a7fd5b283d2de9b/B, priority=13, startTime=1731752787912; duration=0sec 2024-11-16T10:26:27,952 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:26:27,952 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7636762808bf3c8f7a7fd5b283d2de9b:B 2024-11-16T10:26:27,952 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T10:26:27,955 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T10:26:27,955 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): 7636762808bf3c8f7a7fd5b283d2de9b/C is initiating minor compaction (all files) 2024-11-16T10:26:27,955 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7636762808bf3c8f7a7fd5b283d2de9b/C in TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:27,955 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/a35d900ddb404b74a4a809dc54f055bc, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/72181371e60b4d158fc347ddf58068fd, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/538f245882584e1c937764c6f4966695, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/e530a53e0aa7480880be996bb54b574f] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp, totalSize=48.1 K 2024-11-16T10:26:27,956 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting a35d900ddb404b74a4a809dc54f055bc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731752782857 2024-11-16T10:26:27,956 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 72181371e60b4d158fc347ddf58068fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731752783425 2024-11-16T10:26:27,956 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 538f245882584e1c937764c6f4966695, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731752783500 2024-11-16T10:26:27,957 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting e530a53e0aa7480880be996bb54b574f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731752785625 2024-11-16T10:26:27,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167fdc1696bc5c4d1c8c2d3e0c9aea8477_7636762808bf3c8f7a7fd5b283d2de9b is 50, key is test_row_0/A:col10/1731752786273/Put/seqid=0 2024-11-16T10:26:27,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742397_1573 (size=4469) 2024-11-16T10:26:27,994 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7636762808bf3c8f7a7fd5b283d2de9b#C#compaction#487 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:26:27,994 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/3218f6c44925467bb97aa58fff297f85 is 50, key is test_row_0/C:col10/1731752786256/Put/seqid=0 2024-11-16T10:26:28,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742398_1574 (size=12454) 2024-11-16T10:26:28,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742399_1575 (size=12915) 2024-11-16T10:26:28,035 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/3218f6c44925467bb97aa58fff297f85 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/3218f6c44925467bb97aa58fff297f85 2024-11-16T10:26:28,043 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7636762808bf3c8f7a7fd5b283d2de9b/C of 7636762808bf3c8f7a7fd5b283d2de9b into 3218f6c44925467bb97aa58fff297f85(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:26:28,043 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:28,043 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., storeName=7636762808bf3c8f7a7fd5b283d2de9b/C, priority=12, startTime=1731752787912; duration=0sec 2024-11-16T10:26:28,043 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:28,043 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7636762808bf3c8f7a7fd5b283d2de9b:C 2024-11-16T10:26:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T10:26:28,382 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7636762808bf3c8f7a7fd5b283d2de9b#A#compaction#485 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:26:28,383 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/8fdc951a002c48e990cd346af5b8b567 is 175, key is test_row_0/A:col10/1731752786256/Put/seqid=0 2024-11-16T10:26:28,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742400_1576 (size=31903) 2024-11-16T10:26:28,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:26:28,417 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167fdc1696bc5c4d1c8c2d3e0c9aea8477_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167fdc1696bc5c4d1c8c2d3e0c9aea8477_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:28,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/92887422a9ce49c0aa09c8465560b758, store: [table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:28,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/92887422a9ce49c0aa09c8465560b758 is 175, key is test_row_0/A:col10/1731752786273/Put/seqid=0 2024-11-16T10:26:28,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742401_1577 (size=31255) 2024-11-16T10:26:28,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:28,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:28,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752848478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:28,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:28,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:28,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:28,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752848482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:28,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:28,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752848586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:28,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:28,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:28,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:28,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752848604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:28,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:28,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:28,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T10:26:28,793 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/8fdc951a002c48e990cd346af5b8b567 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/8fdc951a002c48e990cd346af5b8b567 2024-11-16T10:26:28,796 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7636762808bf3c8f7a7fd5b283d2de9b/A of 7636762808bf3c8f7a7fd5b283d2de9b into 8fdc951a002c48e990cd346af5b8b567(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:26:28,796 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:28,796 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., storeName=7636762808bf3c8f7a7fd5b283d2de9b/A, priority=13, startTime=1731752787912; duration=0sec 2024-11-16T10:26:28,796 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:28,796 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7636762808bf3c8f7a7fd5b283d2de9b:A 2024-11-16T10:26:28,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752848796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:28,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752848816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:28,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:28,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:28,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:28,822 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/92887422a9ce49c0aa09c8465560b758 2024-11-16T10:26:28,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/5a9539e2d117442793e25e5f173f3c4e is 50, key is test_row_0/B:col10/1731752786273/Put/seqid=0 2024-11-16T10:26:28,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742402_1578 (size=12301) 2024-11-16T10:26:29,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752849106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:29,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:29,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:29,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:29,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752849126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:29,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:29,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:29,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:29,238 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/5a9539e2d117442793e25e5f173f3c4e 2024-11-16T10:26:29,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/12dc20b112f74fdfa83bdd1eaa401c31 is 50, key is test_row_0/C:col10/1731752786273/Put/seqid=0 2024-11-16T10:26:29,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742403_1579 (size=12301) 2024-11-16T10:26:29,248 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/12dc20b112f74fdfa83bdd1eaa401c31 2024-11-16T10:26:29,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/92887422a9ce49c0aa09c8465560b758 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/92887422a9ce49c0aa09c8465560b758 2024-11-16T10:26:29,256 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/92887422a9ce49c0aa09c8465560b758, entries=150, sequenceid=289, filesize=30.5 K 2024-11-16T10:26:29,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/5a9539e2d117442793e25e5f173f3c4e as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/5a9539e2d117442793e25e5f173f3c4e 2024-11-16T10:26:29,260 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/5a9539e2d117442793e25e5f173f3c4e, entries=150, sequenceid=289, filesize=12.0 K 2024-11-16T10:26:29,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/12dc20b112f74fdfa83bdd1eaa401c31 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/12dc20b112f74fdfa83bdd1eaa401c31 2024-11-16T10:26:29,265 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/12dc20b112f74fdfa83bdd1eaa401c31, entries=150, sequenceid=289, filesize=12.0 K 2024-11-16T10:26:29,266 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 7636762808bf3c8f7a7fd5b283d2de9b in 1328ms, sequenceid=289, compaction requested=false 2024-11-16T10:26:29,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:29,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:29,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-16T10:26:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-16T10:26:29,269 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-16T10:26:29,269 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6340 sec 2024-11-16T10:26:29,270 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.6380 sec 2024-11-16T10:26:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:29,619 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:26:29,619 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7636762808bf3c8f7a7fd5b283d2de9b 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-16T10:26:29,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116cea22c88d5c045799cabd5a17c7cab1e_7636762808bf3c8f7a7fd5b283d2de9b is 50, key is test_row_0/A:col10/1731752788480/Put/seqid=0 2024-11-16T10:26:29,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742404_1580 (size=14994) 2024-11-16T10:26:29,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752849636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:29,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:29,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:29,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:29,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752849639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:29,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:29,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:29,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:29,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:29,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752849746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:29,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:29,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:29,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T10:26:29,777 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:26:29,777 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T10:26:29,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-16T10:26:29,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T10:26:29,778 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T10:26:29,779 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T10:26:29,779 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T10:26:29,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T10:26:29,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T10:26:29,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:29,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:29,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:29,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:29,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:29,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:29,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:29,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752849956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:29,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:29,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:29,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:30,033 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:26:30,036 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116cea22c88d5c045799cabd5a17c7cab1e_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116cea22c88d5c045799cabd5a17c7cab1e_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:30,036 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/28d3d864e43640dc8fab06e437a01a6b, store: [table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:30,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/28d3d864e43640dc8fab06e437a01a6b is 175, key is test_row_0/A:col10/1731752788480/Put/seqid=0 2024-11-16T10:26:30,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742405_1581 (size=39949) 2024-11-16T10:26:30,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T10:26:30,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:30,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:30,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:30,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:30,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:30,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:30,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T10:26:30,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T10:26:30,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:30,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:30,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:30,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:30,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:30,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:30,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752850266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:30,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:30,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:30,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:30,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T10:26:30,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:30,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:30,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:30,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:30,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:30,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:30,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T10:26:30,442 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=319, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/28d3d864e43640dc8fab06e437a01a6b 2024-11-16T10:26:30,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/ea75ed80c92549e28d877a8a969a0794 is 50, key is test_row_0/B:col10/1731752788480/Put/seqid=0 2024-11-16T10:26:30,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742406_1582 (size=12301) 2024-11-16T10:26:30,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/ea75ed80c92549e28d877a8a969a0794 2024-11-16T10:26:30,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/8c09915670d346c7a63ca429717df969 is 50, key is test_row_0/C:col10/1731752788480/Put/seqid=0 2024-11-16T10:26:30,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742407_1583 (size=12301) 2024-11-16T10:26:30,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T10:26:30,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:30,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:30,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:30,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:30,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:30,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:30,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1731752850656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:30,658 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_0' in 7636762808bf3c8f7a7fd5b283d2de9b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:26:30,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:30,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:30,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:30,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T10:26:30,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:30,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:30,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:30,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:30,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:30,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:30,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:30,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752850777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:30,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:30,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:30,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:30,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T10:26:30,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:30,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:30,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:30,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:30,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:30,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T10:26:30,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/8c09915670d346c7a63ca429717df969 2024-11-16T10:26:30,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/28d3d864e43640dc8fab06e437a01a6b as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/28d3d864e43640dc8fab06e437a01a6b 2024-11-16T10:26:30,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/28d3d864e43640dc8fab06e437a01a6b, entries=200, sequenceid=319, filesize=39.0 K 2024-11-16T10:26:30,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/ea75ed80c92549e28d877a8a969a0794 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/ea75ed80c92549e28d877a8a969a0794 2024-11-16T10:26:30,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/ea75ed80c92549e28d877a8a969a0794, entries=150, sequenceid=319, filesize=12.0 K 2024-11-16T10:26:30,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/8c09915670d346c7a63ca429717df969 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/8c09915670d346c7a63ca429717df969 2024-11-16T10:26:30,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/8c09915670d346c7a63ca429717df969, entries=150, sequenceid=319, filesize=12.0 K 2024-11-16T10:26:30,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 7636762808bf3c8f7a7fd5b283d2de9b in 1285ms, sequenceid=319, compaction requested=true 2024-11-16T10:26:30,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:30,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7636762808bf3c8f7a7fd5b283d2de9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:26:30,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:30,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7636762808bf3c8f7a7fd5b283d2de9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:26:30,904 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:26:30,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:26:30,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7636762808bf3c8f7a7fd5b283d2de9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T10:26:30,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T10:26:30,904 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:26:30,906 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:26:30,906 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:26:30,906 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1541): 7636762808bf3c8f7a7fd5b283d2de9b/A is initiating minor compaction (all files) 2024-11-16T10:26:30,906 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7636762808bf3c8f7a7fd5b283d2de9b/A in TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:30,906 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/8fdc951a002c48e990cd346af5b8b567, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/92887422a9ce49c0aa09c8465560b758, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/28d3d864e43640dc8fab06e437a01a6b] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp, totalSize=100.7 K 2024-11-16T10:26:30,906 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:30,906 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. files: [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/8fdc951a002c48e990cd346af5b8b567, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/92887422a9ce49c0aa09c8465560b758, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/28d3d864e43640dc8fab06e437a01a6b] 2024-11-16T10:26:30,907 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): 7636762808bf3c8f7a7fd5b283d2de9b/B is initiating minor compaction (all files) 2024-11-16T10:26:30,907 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7636762808bf3c8f7a7fd5b283d2de9b/B in TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:30,907 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/54205d9dc9724cb6a80f358332b98251, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/5a9539e2d117442793e25e5f173f3c4e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/ea75ed80c92549e28d877a8a969a0794] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp, totalSize=36.7 K 2024-11-16T10:26:30,907 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 54205d9dc9724cb6a80f358332b98251, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731752785625 2024-11-16T10:26:30,907 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fdc951a002c48e990cd346af5b8b567, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731752785625 2024-11-16T10:26:30,907 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a9539e2d117442793e25e5f173f3c4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731752786269 2024-11-16T10:26:30,907 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92887422a9ce49c0aa09c8465560b758, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731752786269 2024-11-16T10:26:30,907 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting ea75ed80c92549e28d877a8a969a0794, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731752788471 2024-11-16T10:26:30,908 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28d3d864e43640dc8fab06e437a01a6b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731752788466 2024-11-16T10:26:30,913 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7636762808bf3c8f7a7fd5b283d2de9b#B#compaction#493 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:26:30,913 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/9bd8f8ef793340788c92ba788e9451ec is 50, key is test_row_0/B:col10/1731752788480/Put/seqid=0 2024-11-16T10:26:30,915 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:30,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T10:26:30,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742408_1584 (size=13051) 2024-11-16T10:26:30,925 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/9bd8f8ef793340788c92ba788e9451ec as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/9bd8f8ef793340788c92ba788e9451ec 2024-11-16T10:26:30,929 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116f755661a66034d7d86043bb3d71c68f6_7636762808bf3c8f7a7fd5b283d2de9b store=[table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:30,930 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116f755661a66034d7d86043bb3d71c68f6_7636762808bf3c8f7a7fd5b283d2de9b, store=[table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:30,931 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f755661a66034d7d86043bb3d71c68f6_7636762808bf3c8f7a7fd5b283d2de9b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:30,931 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7636762808bf3c8f7a7fd5b283d2de9b/B of 7636762808bf3c8f7a7fd5b283d2de9b into 9bd8f8ef793340788c92ba788e9451ec(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:26:30,931 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:30,931 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., storeName=7636762808bf3c8f7a7fd5b283d2de9b/B, priority=13, startTime=1731752790904; duration=0sec 2024-11-16T10:26:30,931 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T10:26:30,931 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7636762808bf3c8f7a7fd5b283d2de9b:B 2024-11-16T10:26:30,931 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T10:26:30,932 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T10:26:30,933 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1541): 7636762808bf3c8f7a7fd5b283d2de9b/C is initiating minor compaction (all files) 2024-11-16T10:26:30,933 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7636762808bf3c8f7a7fd5b283d2de9b/C in TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:30,933 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/3218f6c44925467bb97aa58fff297f85, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/12dc20b112f74fdfa83bdd1eaa401c31, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/8c09915670d346c7a63ca429717df969] into tmpdir=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp, totalSize=36.6 K 2024-11-16T10:26:30,933 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 3218f6c44925467bb97aa58fff297f85, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731752785625 2024-11-16T10:26:30,933 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 12dc20b112f74fdfa83bdd1eaa401c31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731752786269 2024-11-16T10:26:30,933 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c09915670d346c7a63ca429717df969, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731752788471 2024-11-16T10:26:30,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742409_1585 (size=4469) 2024-11-16T10:26:30,945 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7636762808bf3c8f7a7fd5b283d2de9b#C#compaction#495 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:26:30,946 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/f5581450ee924a46969d172aa449bbab is 50, key is test_row_0/C:col10/1731752788480/Put/seqid=0 2024-11-16T10:26:30,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742410_1586 (size=13017) 2024-11-16T10:26:30,953 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/f5581450ee924a46969d172aa449bbab as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/f5581450ee924a46969d172aa449bbab 2024-11-16T10:26:30,958 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7636762808bf3c8f7a7fd5b283d2de9b/C of 7636762808bf3c8f7a7fd5b283d2de9b into f5581450ee924a46969d172aa449bbab(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:26:30,958 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:30,958 INFO [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., storeName=7636762808bf3c8f7a7fd5b283d2de9b/C, priority=13, startTime=1731752790904; duration=0sec 2024-11-16T10:26:30,958 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:30,958 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7636762808bf3c8f7a7fd5b283d2de9b:C 2024-11-16T10:26:31,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T10:26:31,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:31,004 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 7636762808bf3c8f7a7fd5b283d2de9b 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-16T10:26:31,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168a61d04cec5f42729f48d9c98c7273a9_7636762808bf3c8f7a7fd5b283d2de9b is 50, key is test_row_0/A:col10/1731752789638/Put/seqid=0 2024-11-16T10:26:31,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742411_1587 (size=12454) 2024-11-16T10:26:31,342 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7636762808bf3c8f7a7fd5b283d2de9b#A#compaction#494 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T10:26:31,343 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/174908864f164ca0b642d1394f9692a2 is 175, key is test_row_0/A:col10/1731752788480/Put/seqid=0 2024-11-16T10:26:31,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742412_1588 (size=32005) 2024-11-16T10:26:31,351 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/174908864f164ca0b642d1394f9692a2 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/174908864f164ca0b642d1394f9692a2 2024-11-16T10:26:31,355 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7636762808bf3c8f7a7fd5b283d2de9b/A of 7636762808bf3c8f7a7fd5b283d2de9b into 174908864f164ca0b642d1394f9692a2(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T10:26:31,355 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:31,356 INFO [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., storeName=7636762808bf3c8f7a7fd5b283d2de9b/A, priority=13, startTime=1731752790904; duration=0sec 2024-11-16T10:26:31,356 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:31,356 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7636762808bf3c8f7a7fd5b283d2de9b:A 2024-11-16T10:26:31,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:26:31,422 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168a61d04cec5f42729f48d9c98c7273a9_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168a61d04cec5f42729f48d9c98c7273a9_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:31,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/ceba87b32b1e4a8bb89fb261648b0111, store: [table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:31,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/ceba87b32b1e4a8bb89fb261648b0111 is 175, key is test_row_0/A:col10/1731752789638/Put/seqid=0 2024-11-16T10:26:31,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742413_1589 (size=31255) 2024-11-16T10:26:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:31,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. as already flushing 2024-11-16T10:26:31,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:31,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752851735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:31,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:31,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:31,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752851735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:31,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:31,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:31,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:31,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:31,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:31,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:31,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752851738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:31,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:31,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:31,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:31,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:31,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39146 deadline: 1731752851796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:31,798 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b16ff38458c1,35873,1731752627506 for 'test_row_0' in 7636762808bf3c8f7a7fd5b283d2de9b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T10:26:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:31,831 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=330, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/ceba87b32b1e4a8bb89fb261648b0111 2024-11-16T10:26:31,832 INFO [Thread-2246 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2246. 2024-11-16T10:26:31,832 DEBUG [Thread-2246 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:26:31,832 DEBUG [Thread-2246 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:31,833 DEBUG [Thread-2246 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:31,833 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:26:31,833 INFO [Thread-2244 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2244. 2024-11-16T10:26:31,833 DEBUG [Thread-2244 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:26:31,833 DEBUG [Thread-2244 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:31,833 DEBUG [Thread-2244 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:31,833 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:26:31,835 INFO [Thread-2248 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2248. 2024-11-16T10:26:31,835 DEBUG [Thread-2248 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:26:31,835 DEBUG [Thread-2248 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:31,835 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:26:31,835 DEBUG [Thread-2248 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:31,836 INFO [Thread-2242 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2242. 2024-11-16T10:26:31,836 DEBUG [Thread-2242 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:26:31,836 DEBUG [Thread-2242 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:31,836 DEBUG [Thread-2242 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:31,836 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:26:31,836 INFO [Thread-2240 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2240. 2024-11-16T10:26:31,836 DEBUG [Thread-2240 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:26:31,836 DEBUG [Thread-2240 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:31,836 DEBUG [Thread-2240 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:31,836 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:26:31,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/171ead660c9b49a3b940a59e067df4f1 is 50, key is test_row_0/B:col10/1731752789638/Put/seqid=0 2024-11-16T10:26:31,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742414_1590 (size=12301) 2024-11-16T10:26:31,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:31,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:31,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:31,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752851846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:31,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752851846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:31,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752851846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:31,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T10:26:32,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:32,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:32,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:32,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752852056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:32,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752852056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:32,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752852057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:32,240 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/171ead660c9b49a3b940a59e067df4f1 2024-11-16T10:26:32,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/4d3de424afb54ecf82d171e72570eb20 is 50, key is test_row_0/C:col10/1731752789638/Put/seqid=0 2024-11-16T10:26:32,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742415_1591 (size=12301) 2024-11-16T10:26:32,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:32,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:32,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T10:26:32,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39202 deadline: 1731752852366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:32,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39142 deadline: 1731752852366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:32,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39204 deadline: 1731752852367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:32,367 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:32,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:32,367 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 , the old value is region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:32,367 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:32,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:32,367 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T10:26:32,367 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:32,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:32,367 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., hostname=b16ff38458c1,35873,1731752627506, seqNum=5 because the exception is null or not the one we care about 2024-11-16T10:26:32,648 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/4d3de424afb54ecf82d171e72570eb20 2024-11-16T10:26:32,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/ceba87b32b1e4a8bb89fb261648b0111 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/ceba87b32b1e4a8bb89fb261648b0111 2024-11-16T10:26:32,654 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/ceba87b32b1e4a8bb89fb261648b0111, entries=150, sequenceid=330, filesize=30.5 K 2024-11-16T10:26:32,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/171ead660c9b49a3b940a59e067df4f1 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/171ead660c9b49a3b940a59e067df4f1 2024-11-16T10:26:32,657 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/171ead660c9b49a3b940a59e067df4f1, entries=150, sequenceid=330, filesize=12.0 K 2024-11-16T10:26:32,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/4d3de424afb54ecf82d171e72570eb20 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/4d3de424afb54ecf82d171e72570eb20 2024-11-16T10:26:32,660 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/4d3de424afb54ecf82d171e72570eb20, entries=150, sequenceid=330, filesize=12.0 K 2024-11-16T10:26:32,660 INFO [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 7636762808bf3c8f7a7fd5b283d2de9b in 1656ms, sequenceid=330, compaction requested=false 2024-11-16T10:26:32,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:32,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:32,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b16ff38458c1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-16T10:26:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-16T10:26:32,663 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-16T10:26:32,663 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8830 sec 2024-11-16T10:26:32,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.8860 sec 2024-11-16T10:26:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35873 {}] regionserver.HRegion(8855): Flush requested on 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:32,667 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T10:26:32,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7636762808bf3c8f7a7fd5b283d2de9b 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-16T10:26:32,667 INFO [Thread-2229 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2229. 2024-11-16T10:26:32,667 DEBUG [Thread-2229 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:26:32,667 DEBUG [Thread-2229 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:32,667 DEBUG [Thread-2229 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:32,667 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:26:32,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169451ac7a3b6247978e85ca0e79d29304_7636762808bf3c8f7a7fd5b283d2de9b is 50, key is test_row_0/A:col10/1731752792666/Put/seqid=0 2024-11-16T10:26:32,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742416_1592 (size=12454) 2024-11-16T10:26:32,878 INFO [Thread-2237 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2237. 2024-11-16T10:26:32,878 INFO [Thread-2233 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2233. 2024-11-16T10:26:32,878 INFO [Thread-2231 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2231. 2024-11-16T10:26:32,878 DEBUG [Thread-2237 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:26:32,878 DEBUG [Thread-2233 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:26:32,878 DEBUG [Thread-2231 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:26:32,878 DEBUG [Thread-2237 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:32,878 DEBUG [Thread-2233 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:32,878 DEBUG [Thread-2231 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:32,878 DEBUG [Thread-2233 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:32,878 DEBUG [Thread-2237 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:32,878 DEBUG [Thread-2231 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:32,878 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:26:32,878 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:26:32,878 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:26:33,075 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:26:33,077 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169451ac7a3b6247978e85ca0e79d29304_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169451ac7a3b6247978e85ca0e79d29304_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:33,078 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/d6fd7761074147b4a3f8233b0aa2e87c, store: [table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:33,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/d6fd7761074147b4a3f8233b0aa2e87c is 175, key is test_row_0/A:col10/1731752792666/Put/seqid=0 2024-11-16T10:26:33,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742417_1593 (size=31255) 2024-11-16T10:26:33,482 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/d6fd7761074147b4a3f8233b0aa2e87c 2024-11-16T10:26:33,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/e2f1316b552440de83a95c091a9a5035 is 50, key is test_row_0/B:col10/1731752792666/Put/seqid=0 2024-11-16T10:26:33,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742418_1594 (size=12301) 2024-11-16T10:26:33,828 INFO [Thread-2235 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2235. 2024-11-16T10:26:33,828 DEBUG [Thread-2235 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T10:26:33,828 DEBUG [Thread-2235 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:33,828 DEBUG [Thread-2235 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:33,828 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:26:33,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/e2f1316b552440de83a95c091a9a5035 2024-11-16T10:26:33,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/2a51d7f836184a488ad13abf450487c6 is 50, key is test_row_0/C:col10/1731752792666/Put/seqid=0 2024-11-16T10:26:33,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742419_1595 (size=12301) 2024-11-16T10:26:33,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T10:26:33,957 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:26:33,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T10:26:33,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-16T10:26:33,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-16T10:26:33,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-16T10:26:33,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-16T10:26:33,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-16T10:26:33,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T10:26:33,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3782 2024-11-16T10:26:33,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3918 2024-11-16T10:26:33,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3846 2024-11-16T10:26:33,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3781 2024-11-16T10:26:33,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3914 2024-11-16T10:26:33,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T10:26:33,958 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T10:26:33,958 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T10:26:33,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:33,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:33,958 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:26:33,958 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T10:26:33,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T10:26:33,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T10:26:33,961 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752793961"}]},"ts":"1731752793961"} 2024-11-16T10:26:33,962 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T10:26:33,962 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T10:26:33,962 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T10:26:33,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7636762808bf3c8f7a7fd5b283d2de9b, UNASSIGN}] 2024-11-16T10:26:33,964 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7636762808bf3c8f7a7fd5b283d2de9b, UNASSIGN 2024-11-16T10:26:33,964 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=7636762808bf3c8f7a7fd5b283d2de9b, regionState=CLOSING, regionLocation=b16ff38458c1,35873,1731752627506 2024-11-16T10:26:33,965 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7636762808bf3c8f7a7fd5b283d2de9b, UNASSIGN because future has completed 2024-11-16T10:26:33,966 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T10:26:33,966 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506}] 2024-11-16T10:26:34,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T10:26:34,118 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(122): Close 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:34,118 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T10:26:34,118 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1722): Closing 7636762808bf3c8f7a7fd5b283d2de9b, disabling compactions & flushes 2024-11-16T10:26:34,118 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:34,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T10:26:34,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/2a51d7f836184a488ad13abf450487c6 2024-11-16T10:26:34,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/d6fd7761074147b4a3f8233b0aa2e87c as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/d6fd7761074147b4a3f8233b0aa2e87c 2024-11-16T10:26:34,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/d6fd7761074147b4a3f8233b0aa2e87c, entries=150, sequenceid=359, filesize=30.5 K 2024-11-16T10:26:34,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/e2f1316b552440de83a95c091a9a5035 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/e2f1316b552440de83a95c091a9a5035 2024-11-16T10:26:34,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/e2f1316b552440de83a95c091a9a5035, entries=150, sequenceid=359, filesize=12.0 K 2024-11-16T10:26:34,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/2a51d7f836184a488ad13abf450487c6 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/2a51d7f836184a488ad13abf450487c6 2024-11-16T10:26:34,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/2a51d7f836184a488ad13abf450487c6, entries=150, sequenceid=359, filesize=12.0 K 2024-11-16T10:26:34,312 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=26.84 KB/27480 for 7636762808bf3c8f7a7fd5b283d2de9b in 1645ms, sequenceid=359, compaction requested=true 2024-11-16T10:26:34,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7636762808bf3c8f7a7fd5b283d2de9b: 2024-11-16T10:26:34,312 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:34,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7636762808bf3c8f7a7fd5b283d2de9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T10:26:34,312 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. because compaction request was cancelled 2024-11-16T10:26:34,312 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:34,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:34,312 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. after waiting 0 ms 2024-11-16T10:26:34,312 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7636762808bf3c8f7a7fd5b283d2de9b:A 2024-11-16T10:26:34,312 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:34,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7636762808bf3c8f7a7fd5b283d2de9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:26:34,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:34,313 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. because compaction request was cancelled 2024-11-16T10:26:34,313 DEBUG [RS:0;b16ff38458c1:35873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7636762808bf3c8f7a7fd5b283d2de9b:B 2024-11-16T10:26:34,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7636762808bf3c8f7a7fd5b283d2de9b:C, priority=-2147483648, current under compaction store size is 2 2024-11-16T10:26:34,313 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(2902): Flushing 7636762808bf3c8f7a7fd5b283d2de9b 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-16T10:26:34,313 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. because compaction request was cancelled 2024-11-16T10:26:34,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T10:26:34,313 DEBUG [RS:0;b16ff38458c1:35873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7636762808bf3c8f7a7fd5b283d2de9b:C 2024-11-16T10:26:34,316 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160735b88b25be4c3d987225ed0eefbb09_7636762808bf3c8f7a7fd5b283d2de9b is 50, key is test_row_0/A:col10/1731752793827/Put/seqid=0 2024-11-16T10:26:34,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742420_1596 (size=9914) 2024-11-16T10:26:34,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T10:26:34,726 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T10:26:34,729 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160735b88b25be4c3d987225ed0eefbb09_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160735b88b25be4c3d987225ed0eefbb09_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:34,730 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/b17cee767a4847bca4342bd00d0ae25b, store: [table=TestAcidGuarantees family=A region=7636762808bf3c8f7a7fd5b283d2de9b] 2024-11-16T10:26:34,730 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/b17cee767a4847bca4342bd00d0ae25b is 175, key is test_row_0/A:col10/1731752793827/Put/seqid=0 2024-11-16T10:26:34,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742421_1597 (size=22561) 2024-11-16T10:26:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T10:26:35,133 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=366, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/b17cee767a4847bca4342bd00d0ae25b 2024-11-16T10:26:35,139 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/50902478676a4a56ae57f968c08ad170 is 50, key is test_row_0/B:col10/1731752793827/Put/seqid=0 2024-11-16T10:26:35,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742422_1598 (size=9857) 2024-11-16T10:26:35,542 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/50902478676a4a56ae57f968c08ad170 2024-11-16T10:26:35,548 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/20e5eb7fb285479c98cc32d5de3aa2d6 is 50, key is test_row_0/C:col10/1731752793827/Put/seqid=0 2024-11-16T10:26:35,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742423_1599 (size=9857) 2024-11-16T10:26:35,951 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/20e5eb7fb285479c98cc32d5de3aa2d6 2024-11-16T10:26:35,955 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/A/b17cee767a4847bca4342bd00d0ae25b as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/b17cee767a4847bca4342bd00d0ae25b 2024-11-16T10:26:35,958 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/b17cee767a4847bca4342bd00d0ae25b, entries=100, sequenceid=366, filesize=22.0 K 2024-11-16T10:26:35,958 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/B/50902478676a4a56ae57f968c08ad170 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/50902478676a4a56ae57f968c08ad170 2024-11-16T10:26:35,961 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/50902478676a4a56ae57f968c08ad170, entries=100, sequenceid=366, filesize=9.6 K 2024-11-16T10:26:35,961 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/.tmp/C/20e5eb7fb285479c98cc32d5de3aa2d6 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/20e5eb7fb285479c98cc32d5de3aa2d6 2024-11-16T10:26:35,964 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/20e5eb7fb285479c98cc32d5de3aa2d6, entries=100, sequenceid=366, filesize=9.6 K 2024-11-16T10:26:35,965 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 7636762808bf3c8f7a7fd5b283d2de9b in 1653ms, sequenceid=366, compaction requested=true 2024-11-16T10:26:35,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/98b784610d31404998f549ae1ce511fb, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/d2d70de2231641f4bec8355666552ffc, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/11b8b65bf0c44f59b582277cced726dd, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/68545189aac84157b9d52afee5afe1d6, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/161841f9206f47ca8de265820a6dd13e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/f4fa580fb7b842579da583bc6de19956, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/e8f7f7b3e5f141deaec5878cbc4f166a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/e49ad4ebd83047eb8b1df6094d651e61, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/131c384a3e4345c99fc65084fca47f52, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/8c2ec46cf8c44d77b17a9d9d4dea5873, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/bd05e391afc943d695552f5c6adb6b42, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/5b77232498484f77b36399735dffb63a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/fe8fbb0da7d94521a093d2a9fc1083b7, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/f17a2675bd52452c9d5dd43d12fe445f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/ab88024ed76f4d0685333f10157b3f18, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/4a23fe391d414defa995c65d3f2ad3ff, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/d4b8ee34a749495bbf1cdcf59b91c484, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/91543c4049fe4f24a0a062aecf36ea7f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/29643a8b50e24b4897c997c7e8c13598, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/8fdc951a002c48e990cd346af5b8b567, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/92887422a9ce49c0aa09c8465560b758, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/28d3d864e43640dc8fab06e437a01a6b] to archive 2024-11-16T10:26:35,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:26:35,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/98b784610d31404998f549ae1ce511fb to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/98b784610d31404998f549ae1ce511fb 2024-11-16T10:26:35,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/d2d70de2231641f4bec8355666552ffc to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/d2d70de2231641f4bec8355666552ffc 2024-11-16T10:26:35,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/11b8b65bf0c44f59b582277cced726dd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/11b8b65bf0c44f59b582277cced726dd 2024-11-16T10:26:35,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/68545189aac84157b9d52afee5afe1d6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/68545189aac84157b9d52afee5afe1d6 2024-11-16T10:26:35,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/161841f9206f47ca8de265820a6dd13e to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/161841f9206f47ca8de265820a6dd13e 2024-11-16T10:26:35,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/f4fa580fb7b842579da583bc6de19956 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/f4fa580fb7b842579da583bc6de19956 2024-11-16T10:26:35,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/e8f7f7b3e5f141deaec5878cbc4f166a to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/e8f7f7b3e5f141deaec5878cbc4f166a 2024-11-16T10:26:35,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/e49ad4ebd83047eb8b1df6094d651e61 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/e49ad4ebd83047eb8b1df6094d651e61 2024-11-16T10:26:35,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/131c384a3e4345c99fc65084fca47f52 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/131c384a3e4345c99fc65084fca47f52 2024-11-16T10:26:35,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/8c2ec46cf8c44d77b17a9d9d4dea5873 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/8c2ec46cf8c44d77b17a9d9d4dea5873 2024-11-16T10:26:35,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/bd05e391afc943d695552f5c6adb6b42 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/bd05e391afc943d695552f5c6adb6b42 2024-11-16T10:26:35,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/5b77232498484f77b36399735dffb63a to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/5b77232498484f77b36399735dffb63a 2024-11-16T10:26:35,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/fe8fbb0da7d94521a093d2a9fc1083b7 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/fe8fbb0da7d94521a093d2a9fc1083b7 2024-11-16T10:26:35,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/f17a2675bd52452c9d5dd43d12fe445f to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/f17a2675bd52452c9d5dd43d12fe445f 2024-11-16T10:26:35,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/ab88024ed76f4d0685333f10157b3f18 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/ab88024ed76f4d0685333f10157b3f18 2024-11-16T10:26:35,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/4a23fe391d414defa995c65d3f2ad3ff to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/4a23fe391d414defa995c65d3f2ad3ff 2024-11-16T10:26:35,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/d4b8ee34a749495bbf1cdcf59b91c484 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/d4b8ee34a749495bbf1cdcf59b91c484 2024-11-16T10:26:35,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/91543c4049fe4f24a0a062aecf36ea7f to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/91543c4049fe4f24a0a062aecf36ea7f 2024-11-16T10:26:35,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/29643a8b50e24b4897c997c7e8c13598 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/29643a8b50e24b4897c997c7e8c13598 2024-11-16T10:26:35,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/8fdc951a002c48e990cd346af5b8b567 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/8fdc951a002c48e990cd346af5b8b567 2024-11-16T10:26:35,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/92887422a9ce49c0aa09c8465560b758 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/92887422a9ce49c0aa09c8465560b758 2024-11-16T10:26:35,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/28d3d864e43640dc8fab06e437a01a6b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/28d3d864e43640dc8fab06e437a01a6b 2024-11-16T10:26:35,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/65de4a495e9e4997ac5ee05687db122d, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/8b4e25b3b8ac4fcaa6cfac2b4360ef1a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/8becd93d098149bba8ce74fb444c4ddf, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/cd91782b76a34caebe2952a822ca94fd, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/f78bc71134e74c1daec893554ed33a83, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/e0caae33f41e4a4cad87152d44f25a88, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/833d7f97e2b54668a6b590c81c94323f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/328dcf3d02cb4400ace823b4283ff85c, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/e1e542a0367e41888121e5ba044cb52e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/464f2392a7f84c658d9280ed04eccb44, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/48e3dbf654cb43d892da301c1b5e2428, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/9af784f4a5dc4629be788b806e3c4fd8, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/fe342fe4975b4f71a35917f07469ed5e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/387f2f01aa1c4f2ca963382a02a583c4, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/d4dcd984ae0d40c593763c04f9c1209c, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/83c5e76f883244ac958035bcef2b7003, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/5cbedee5a89a487f9a8078883595660a, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/ff526256c4f5429081bdc51012c8c6e5, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/54205d9dc9724cb6a80f358332b98251, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/ba9769c2bf0c4e078ecfba87baa9444d, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/5a9539e2d117442793e25e5f173f3c4e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/ea75ed80c92549e28d877a8a969a0794] to archive 2024-11-16T10:26:35,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:26:35,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/65de4a495e9e4997ac5ee05687db122d to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/65de4a495e9e4997ac5ee05687db122d 2024-11-16T10:26:35,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/8b4e25b3b8ac4fcaa6cfac2b4360ef1a to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/8b4e25b3b8ac4fcaa6cfac2b4360ef1a 2024-11-16T10:26:35,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/8becd93d098149bba8ce74fb444c4ddf to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/8becd93d098149bba8ce74fb444c4ddf 2024-11-16T10:26:35,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/cd91782b76a34caebe2952a822ca94fd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/cd91782b76a34caebe2952a822ca94fd 2024-11-16T10:26:35,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/f78bc71134e74c1daec893554ed33a83 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/f78bc71134e74c1daec893554ed33a83 2024-11-16T10:26:35,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/e0caae33f41e4a4cad87152d44f25a88 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/e0caae33f41e4a4cad87152d44f25a88 2024-11-16T10:26:35,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/833d7f97e2b54668a6b590c81c94323f to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/833d7f97e2b54668a6b590c81c94323f 2024-11-16T10:26:35,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/328dcf3d02cb4400ace823b4283ff85c to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/328dcf3d02cb4400ace823b4283ff85c 2024-11-16T10:26:35,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/e1e542a0367e41888121e5ba044cb52e to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/e1e542a0367e41888121e5ba044cb52e 2024-11-16T10:26:35,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/464f2392a7f84c658d9280ed04eccb44 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/464f2392a7f84c658d9280ed04eccb44 2024-11-16T10:26:35,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/48e3dbf654cb43d892da301c1b5e2428 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/48e3dbf654cb43d892da301c1b5e2428 2024-11-16T10:26:35,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/9af784f4a5dc4629be788b806e3c4fd8 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/9af784f4a5dc4629be788b806e3c4fd8 2024-11-16T10:26:35,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/fe342fe4975b4f71a35917f07469ed5e to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/fe342fe4975b4f71a35917f07469ed5e 2024-11-16T10:26:35,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/387f2f01aa1c4f2ca963382a02a583c4 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/387f2f01aa1c4f2ca963382a02a583c4 2024-11-16T10:26:35,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/d4dcd984ae0d40c593763c04f9c1209c to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/d4dcd984ae0d40c593763c04f9c1209c 2024-11-16T10:26:36,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/83c5e76f883244ac958035bcef2b7003 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/83c5e76f883244ac958035bcef2b7003 2024-11-16T10:26:36,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/5cbedee5a89a487f9a8078883595660a to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/5cbedee5a89a487f9a8078883595660a 2024-11-16T10:26:36,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/ff526256c4f5429081bdc51012c8c6e5 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/ff526256c4f5429081bdc51012c8c6e5 2024-11-16T10:26:36,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/54205d9dc9724cb6a80f358332b98251 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/54205d9dc9724cb6a80f358332b98251 2024-11-16T10:26:36,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/ba9769c2bf0c4e078ecfba87baa9444d to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/ba9769c2bf0c4e078ecfba87baa9444d 2024-11-16T10:26:36,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/5a9539e2d117442793e25e5f173f3c4e to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/5a9539e2d117442793e25e5f173f3c4e 2024-11-16T10:26:36,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/ea75ed80c92549e28d877a8a969a0794 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/ea75ed80c92549e28d877a8a969a0794 2024-11-16T10:26:36,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/a4c677056c0c4473a709ec0543e9e591, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/35248701bbbf4bdab1b890f858c2a8cd, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/afce63b7ac4a4ca3a3c9f14ff9d5d860, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/4e5c345fefff4059bdf295c402625a0e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/4da28a9a4b4f412e825992c357a3e369, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/6571a467d21b46479f9dc0591c3b8b57, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/cc941b3146434341a7279ad4a58674a5, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/b95a2c03ad4b4504b6b5523d805016fb, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/1e662aeb69024345ae2f01fb60c02e62, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/646f47c8f3a7431b998a5de6a9ed3947, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/ad729398ca0647e5b1f5ef5beb517528, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/6e27d11518524447beb03468e5028a19, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/4c7f04c7154e4ce899712f180d76414e, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/a35d900ddb404b74a4a809dc54f055bc, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/9e02723e87e8456293b0a6d9cc3bb1fc, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/72181371e60b4d158fc347ddf58068fd, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/538f245882584e1c937764c6f4966695, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/3218f6c44925467bb97aa58fff297f85, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/e530a53e0aa7480880be996bb54b574f, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/12dc20b112f74fdfa83bdd1eaa401c31, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/8c09915670d346c7a63ca429717df969] to archive 2024-11-16T10:26:36,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T10:26:36,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/a4c677056c0c4473a709ec0543e9e591 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/a4c677056c0c4473a709ec0543e9e591 2024-11-16T10:26:36,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/35248701bbbf4bdab1b890f858c2a8cd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/35248701bbbf4bdab1b890f858c2a8cd 2024-11-16T10:26:36,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/afce63b7ac4a4ca3a3c9f14ff9d5d860 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/afce63b7ac4a4ca3a3c9f14ff9d5d860 2024-11-16T10:26:36,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/4e5c345fefff4059bdf295c402625a0e to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/4e5c345fefff4059bdf295c402625a0e 2024-11-16T10:26:36,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/4da28a9a4b4f412e825992c357a3e369 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/4da28a9a4b4f412e825992c357a3e369 2024-11-16T10:26:36,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/6571a467d21b46479f9dc0591c3b8b57 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/6571a467d21b46479f9dc0591c3b8b57 2024-11-16T10:26:36,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/cc941b3146434341a7279ad4a58674a5 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/cc941b3146434341a7279ad4a58674a5 2024-11-16T10:26:36,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/b95a2c03ad4b4504b6b5523d805016fb to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/b95a2c03ad4b4504b6b5523d805016fb 2024-11-16T10:26:36,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/1e662aeb69024345ae2f01fb60c02e62 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/1e662aeb69024345ae2f01fb60c02e62 2024-11-16T10:26:36,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/646f47c8f3a7431b998a5de6a9ed3947 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/646f47c8f3a7431b998a5de6a9ed3947 2024-11-16T10:26:36,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/ad729398ca0647e5b1f5ef5beb517528 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/ad729398ca0647e5b1f5ef5beb517528 2024-11-16T10:26:36,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/6e27d11518524447beb03468e5028a19 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/6e27d11518524447beb03468e5028a19 2024-11-16T10:26:36,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/4c7f04c7154e4ce899712f180d76414e to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/4c7f04c7154e4ce899712f180d76414e 2024-11-16T10:26:36,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/a35d900ddb404b74a4a809dc54f055bc to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/a35d900ddb404b74a4a809dc54f055bc 2024-11-16T10:26:36,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/9e02723e87e8456293b0a6d9cc3bb1fc to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/9e02723e87e8456293b0a6d9cc3bb1fc 2024-11-16T10:26:36,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/72181371e60b4d158fc347ddf58068fd to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/72181371e60b4d158fc347ddf58068fd 2024-11-16T10:26:36,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/538f245882584e1c937764c6f4966695 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/538f245882584e1c937764c6f4966695 2024-11-16T10:26:36,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/3218f6c44925467bb97aa58fff297f85 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/3218f6c44925467bb97aa58fff297f85 2024-11-16T10:26:36,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/e530a53e0aa7480880be996bb54b574f to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/e530a53e0aa7480880be996bb54b574f 2024-11-16T10:26:36,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/12dc20b112f74fdfa83bdd1eaa401c31 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/12dc20b112f74fdfa83bdd1eaa401c31 2024-11-16T10:26:36,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/8c09915670d346c7a63ca429717df969 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/8c09915670d346c7a63ca429717df969 2024-11-16T10:26:36,024 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/recovered.edits/369.seqid, newMaxSeqId=369, maxSeqId=4 2024-11-16T10:26:36,025 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. 2024-11-16T10:26:36,025 DEBUG [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1676): Region close journal for 7636762808bf3c8f7a7fd5b283d2de9b: Waiting for close lock at 1731752794118Running coprocessor pre-close hooks at 1731752794118Disabling compacts and flushes for region at 1731752794118Disabling writes for close at 1731752794312 (+194 ms)Obtaining lock to block concurrent updates at 1731752794313 (+1 ms)Preparing flush snapshotting stores in 7636762808bf3c8f7a7fd5b283d2de9b at 1731752794313Finished memstore snapshotting TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731752794313Flushing stores of TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b. at 1731752794313Flushing 7636762808bf3c8f7a7fd5b283d2de9b/A: creating writer at 1731752794313Flushing mob file 7636762808bf3c8f7a7fd5b283d2de9b/A: appending metadata at 1731752794316 (+3 ms)Flushing mob file 7636762808bf3c8f7a7fd5b283d2de9b/A: closing flushed file at 1731752794316Flushing 7636762808bf3c8f7a7fd5b283d2de9b/A: appending metadata at 1731752794730 (+414 ms)Flushing 7636762808bf3c8f7a7fd5b283d2de9b/A: closing flushed file at 1731752794730Flushing 7636762808bf3c8f7a7fd5b283d2de9b/B: creating writer at 1731752795136 (+406 ms)Flushing 7636762808bf3c8f7a7fd5b283d2de9b/B: appending metadata at 1731752795138 (+2 ms)Flushing 7636762808bf3c8f7a7fd5b283d2de9b/B: closing flushed file at 1731752795138Flushing 7636762808bf3c8f7a7fd5b283d2de9b/C: creating writer at 1731752795545 (+407 ms)Flushing 7636762808bf3c8f7a7fd5b283d2de9b/C: appending metadata at 1731752795547 (+2 ms)Flushing 7636762808bf3c8f7a7fd5b283d2de9b/C: closing flushed file at 1731752795548 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8d202fe: reopening flushed file at 1731752795954 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@734a95af: reopening flushed file at 1731752795958 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43233465: reopening flushed file at 1731752795961 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 7636762808bf3c8f7a7fd5b283d2de9b in 1653ms, sequenceid=366, compaction requested=true at 1731752795965 (+4 ms)Writing region close event to WAL at 1731752796022 (+57 ms)Running coprocessor post-close hooks at 1731752796025 (+3 ms)Closed at 1731752796025 2024-11-16T10:26:36,026 INFO [RS_CLOSE_REGION-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(157): Closed 7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:36,027 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=7636762808bf3c8f7a7fd5b283d2de9b, regionState=CLOSED 2024-11-16T10:26:36,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 because future has completed 2024-11-16T10:26:36,030 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-16T10:26:36,030 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseRegionProcedure 7636762808bf3c8f7a7fd5b283d2de9b, server=b16ff38458c1,35873,1731752627506 in 2.0630 sec 2024-11-16T10:26:36,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-11-16T10:26:36,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7636762808bf3c8f7a7fd5b283d2de9b, UNASSIGN in 2.0670 sec 2024-11-16T10:26:36,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-16T10:26:36,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.0700 sec 2024-11-16T10:26:36,034 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731752796034"}]},"ts":"1731752796034"} 2024-11-16T10:26:36,035 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T10:26:36,035 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T10:26:36,037 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.0780 sec 2024-11-16T10:26:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T10:26:36,109 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:26:36,109 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T10:26:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:26:36,111 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:26:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T10:26:36,111 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=166, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:26:36,113 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:36,115 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A, FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B, FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C, FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/recovered.edits] 2024-11-16T10:26:36,116 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/174908864f164ca0b642d1394f9692a2 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/174908864f164ca0b642d1394f9692a2 2024-11-16T10:26:36,117 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/b17cee767a4847bca4342bd00d0ae25b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/b17cee767a4847bca4342bd00d0ae25b 2024-11-16T10:26:36,118 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/ceba87b32b1e4a8bb89fb261648b0111 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/ceba87b32b1e4a8bb89fb261648b0111 2024-11-16T10:26:36,119 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/d6fd7761074147b4a3f8233b0aa2e87c to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/A/d6fd7761074147b4a3f8233b0aa2e87c 2024-11-16T10:26:36,120 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/171ead660c9b49a3b940a59e067df4f1 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/171ead660c9b49a3b940a59e067df4f1 2024-11-16T10:26:36,121 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/50902478676a4a56ae57f968c08ad170 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/50902478676a4a56ae57f968c08ad170 2024-11-16T10:26:36,122 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/9bd8f8ef793340788c92ba788e9451ec to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/9bd8f8ef793340788c92ba788e9451ec 2024-11-16T10:26:36,122 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/e2f1316b552440de83a95c091a9a5035 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/B/e2f1316b552440de83a95c091a9a5035 2024-11-16T10:26:36,124 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/20e5eb7fb285479c98cc32d5de3aa2d6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/20e5eb7fb285479c98cc32d5de3aa2d6 2024-11-16T10:26:36,125 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/2a51d7f836184a488ad13abf450487c6 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/2a51d7f836184a488ad13abf450487c6 2024-11-16T10:26:36,125 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/4d3de424afb54ecf82d171e72570eb20 to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/4d3de424afb54ecf82d171e72570eb20 2024-11-16T10:26:36,126 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/f5581450ee924a46969d172aa449bbab to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/C/f5581450ee924a46969d172aa449bbab 2024-11-16T10:26:36,128 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/recovered.edits/369.seqid to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b/recovered.edits/369.seqid 2024-11-16T10:26:36,128 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/default/TestAcidGuarantees/7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:36,128 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T10:26:36,129 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T10:26:36,129 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-16T10:26:36,131 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160385df09e9b34edb98151ffa7e4e1b17_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160385df09e9b34edb98151ffa7e4e1b17_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:36,132 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160735b88b25be4c3d987225ed0eefbb09_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160735b88b25be4c3d987225ed0eefbb09_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:36,133 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116174c08df8adc4e698c18902d8270a7de_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116174c08df8adc4e698c18902d8270a7de_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:36,133 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162557d8d5cdd745b9a9f50dc518316293_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162557d8d5cdd745b9a9f50dc518316293_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:36,134 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162fe23276ddf34663b0c0b94d98303472_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162fe23276ddf34663b0c0b94d98303472_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:36,135 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163c6037c036b44ac49ac7a87924bab792_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163c6037c036b44ac49ac7a87924bab792_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:36,135 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167adfb8c4dc5a4c5c9a806af29499f920_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167adfb8c4dc5a4c5c9a806af29499f920_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:36,136 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167fdc1696bc5c4d1c8c2d3e0c9aea8477_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167fdc1696bc5c4d1c8c2d3e0c9aea8477_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:36,136 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168a61d04cec5f42729f48d9c98c7273a9_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168a61d04cec5f42729f48d9c98c7273a9_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:36,137 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169451ac7a3b6247978e85ca0e79d29304_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169451ac7a3b6247978e85ca0e79d29304_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:36,138 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111699348bd3ab354c598c00ee3133d0b588_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111699348bd3ab354c598c00ee3133d0b588_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:36,138 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169a59dab5729b487b9422e562f02cb784_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169a59dab5729b487b9422e562f02cb784_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:36,139 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116aaac2f53221e4541a8fa741d39d2e1d4_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116aaac2f53221e4541a8fa741d39d2e1d4_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:36,139 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c0e512abb404458e96343a7277de704c_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c0e512abb404458e96343a7277de704c_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:36,140 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c61e4f5c91a540ed86f70ca9a0e47045_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c61e4f5c91a540ed86f70ca9a0e47045_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:36,141 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116cea22c88d5c045799cabd5a17c7cab1e_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116cea22c88d5c045799cabd5a17c7cab1e_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:36,141 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e0ef16b3c98d433ea62b550fb39e752e_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e0ef16b3c98d433ea62b550fb39e752e_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:36,142 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e8ae80f1bbae4570a732d56f349c3b19_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e8ae80f1bbae4570a732d56f349c3b19_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:36,143 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fa7f0d62b1714f9f94dc8aced44c576a_7636762808bf3c8f7a7fd5b283d2de9b to hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fa7f0d62b1714f9f94dc8aced44c576a_7636762808bf3c8f7a7fd5b283d2de9b 2024-11-16T10:26:36,143 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T10:26:36,145 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=166, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:26:36,146 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T10:26:36,148 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T10:26:36,148 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=166, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:26:36,148 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T10:26:36,149 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731752796148"}]},"ts":"9223372036854775807"} 2024-11-16T10:26:36,150 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T10:26:36,150 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7636762808bf3c8f7a7fd5b283d2de9b, NAME => 'TestAcidGuarantees,,1731752768723.7636762808bf3c8f7a7fd5b283d2de9b.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T10:26:36,150 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T10:26:36,150 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731752796150"}]},"ts":"9223372036854775807"} 2024-11-16T10:26:36,151 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T10:26:36,152 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=166, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T10:26:36,153 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 42 msec 2024-11-16T10:26:36,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T10:26:36,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T10:26:36,217 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T10:26:36,224 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240 (was 241), OpenFileDescriptor=447 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=491 (was 541), ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=3828 (was 3842) 2024-11-16T10:26:36,225 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-16T10:26:36,225 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T10:26:36,225 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T10:26:36,225 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:36,225 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:36,225 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T10:26:36,225 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-16T10:26:36,225 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1219616701, stopped=false 2024-11-16T10:26:36,226 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=b16ff38458c1,38809,1731752626047 2024-11-16T10:26:36,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T10:26:36,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35873-0x101432bffa20001, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T10:26:36,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35873-0x101432bffa20001, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:26:36,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:26:36,298 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T10:26:36,299 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35873-0x101432bffa20001, quorum=127.0.0.1:49279, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T10:26:36,299 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T10:26:36,299 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T10:26:36,299 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T10:26:36,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:36,300 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b16ff38458c1,35873,1731752627506' ***** 2024-11-16T10:26:36,300 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T10:26:36,300 INFO [RS:0;b16ff38458c1:35873 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T10:26:36,300 INFO [RS:0;b16ff38458c1:35873 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T10:26:36,300 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T10:26:36,300 INFO [RS:0;b16ff38458c1:35873 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T10:26:36,301 INFO [RS:0;b16ff38458c1:35873 {}] regionserver.HRegionServer(959): stopping server b16ff38458c1,35873,1731752627506 2024-11-16T10:26:36,301 INFO [RS:0;b16ff38458c1:35873 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T10:26:36,301 INFO [RS:0;b16ff38458c1:35873 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;b16ff38458c1:35873. 2024-11-16T10:26:36,301 DEBUG [RS:0;b16ff38458c1:35873 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T10:26:36,301 DEBUG [RS:0;b16ff38458c1:35873 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:36,301 INFO [RS:0;b16ff38458c1:35873 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T10:26:36,301 INFO [RS:0;b16ff38458c1:35873 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T10:26:36,301 INFO [RS:0;b16ff38458c1:35873 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T10:26:36,301 INFO [RS:0;b16ff38458c1:35873 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T10:26:36,302 INFO [RS:0;b16ff38458c1:35873 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-16T10:26:36,302 DEBUG [RS:0;b16ff38458c1:35873 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-16T10:26:36,302 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T10:26:36,302 INFO [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T10:26:36,302 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T10:26:36,302 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T10:26:36,302 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T10:26:36,302 DEBUG [RS:0;b16ff38458c1:35873 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T10:26:36,302 INFO [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-16T10:26:36,321 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/hbase/meta/1588230740/.tmp/info/5bfe6575a5cb4044a727cc22b507dbc3 is 95, key is TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2./info:/1731752657267/DeleteFamily/seqid=0 2024-11-16T10:26:36,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742424_1600 (size=6514) 2024-11-16T10:26:36,341 INFO [regionserver/b16ff38458c1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T10:26:36,495 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T10:26:36,502 DEBUG [RS:0;b16ff38458c1:35873 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T10:26:36,596 INFO [regionserver/b16ff38458c1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-16T10:26:36,596 INFO [regionserver/b16ff38458c1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-16T10:26:36,703 DEBUG [RS:0;b16ff38458c1:35873 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T10:26:36,725 INFO [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/hbase/meta/1588230740/.tmp/info/5bfe6575a5cb4044a727cc22b507dbc3 2024-11-16T10:26:36,741 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/hbase/meta/1588230740/.tmp/ns/1c85a07f4fa240aaa96eb2d655bfa44f is 93, key is TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2./ns:/1731752657228/DeleteFamily/seqid=0 2024-11-16T10:26:36,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742425_1601 (size=5906) 2024-11-16T10:26:36,903 DEBUG [RS:0;b16ff38458c1:35873 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T10:26:37,103 DEBUG [RS:0;b16ff38458c1:35873 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T10:26:37,145 INFO [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/hbase/meta/1588230740/.tmp/ns/1c85a07f4fa240aaa96eb2d655bfa44f 2024-11-16T10:26:37,162 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/hbase/meta/1588230740/.tmp/rep_barrier/9000abfb9a724f9cbed2f02ca58db8c8 is 102, key is TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2./rep_barrier:/1731752657228/DeleteFamily/seqid=0 2024-11-16T10:26:37,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742426_1602 (size=6025) 2024-11-16T10:26:37,303 DEBUG [RS:0;b16ff38458c1:35873 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T10:26:37,503 DEBUG [RS:0;b16ff38458c1:35873 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T10:26:37,565 INFO [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/hbase/meta/1588230740/.tmp/rep_barrier/9000abfb9a724f9cbed2f02ca58db8c8 2024-11-16T10:26:37,582 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/hbase/meta/1588230740/.tmp/table/e8e2885b5f9c4ce48d135a600964257a is 96, key is TestAcidGuarantees,,1731752631650.e63c645d595c80b0e763eedbebd195b2./table:/1731752657228/DeleteFamily/seqid=0 2024-11-16T10:26:37,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742427_1603 (size=5936) 2024-11-16T10:26:37,704 DEBUG [RS:0;b16ff38458c1:35873 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T10:26:37,904 DEBUG [RS:0;b16ff38458c1:35873 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T10:26:37,985 INFO [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/hbase/meta/1588230740/.tmp/table/e8e2885b5f9c4ce48d135a600964257a 2024-11-16T10:26:37,989 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/hbase/meta/1588230740/.tmp/info/5bfe6575a5cb4044a727cc22b507dbc3 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/hbase/meta/1588230740/info/5bfe6575a5cb4044a727cc22b507dbc3 2024-11-16T10:26:37,992 INFO [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/hbase/meta/1588230740/info/5bfe6575a5cb4044a727cc22b507dbc3, entries=12, sequenceid=90, filesize=6.4 K 2024-11-16T10:26:37,993 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/hbase/meta/1588230740/.tmp/ns/1c85a07f4fa240aaa96eb2d655bfa44f as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/hbase/meta/1588230740/ns/1c85a07f4fa240aaa96eb2d655bfa44f 2024-11-16T10:26:37,995 INFO [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/hbase/meta/1588230740/ns/1c85a07f4fa240aaa96eb2d655bfa44f, entries=8, sequenceid=90, filesize=5.8 K 2024-11-16T10:26:37,996 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/hbase/meta/1588230740/.tmp/rep_barrier/9000abfb9a724f9cbed2f02ca58db8c8 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/hbase/meta/1588230740/rep_barrier/9000abfb9a724f9cbed2f02ca58db8c8 2024-11-16T10:26:37,998 INFO [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/hbase/meta/1588230740/rep_barrier/9000abfb9a724f9cbed2f02ca58db8c8, entries=6, sequenceid=90, filesize=5.9 K 2024-11-16T10:26:37,999 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/hbase/meta/1588230740/.tmp/table/e8e2885b5f9c4ce48d135a600964257a as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/hbase/meta/1588230740/table/e8e2885b5f9c4ce48d135a600964257a 2024-11-16T10:26:38,001 INFO [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/hbase/meta/1588230740/table/e8e2885b5f9c4ce48d135a600964257a, entries=7, sequenceid=90, filesize=5.8 K 2024-11-16T10:26:38,002 INFO [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1700ms, sequenceid=90, compaction requested=false 2024-11-16T10:26:38,005 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-16T10:26:38,006 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T10:26:38,006 INFO [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T10:26:38,006 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731752796302Running coprocessor pre-close hooks at 1731752796302Disabling compacts and flushes for region at 1731752796302Disabling writes for close at 1731752796302Obtaining lock to block concurrent updates at 1731752796302Preparing flush snapshotting stores in 1588230740 at 1731752796302Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731752796302Flushing stores of hbase:meta,,1.1588230740 at 1731752796303 (+1 ms)Flushing 1588230740/info: creating writer at 1731752796303Flushing 1588230740/info: appending metadata at 1731752796321 (+18 ms)Flushing 1588230740/info: closing flushed file at 1731752796321Flushing 1588230740/ns: creating writer at 1731752796728 (+407 ms)Flushing 1588230740/ns: appending metadata at 1731752796741 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1731752796741Flushing 1588230740/rep_barrier: creating writer at 1731752797148 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731752797161 (+13 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731752797161Flushing 1588230740/table: creating writer at 1731752797568 (+407 ms)Flushing 1588230740/table: appending metadata at 1731752797581 (+13 ms)Flushing 1588230740/table: closing flushed file at 1731752797581Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@461f4b62: reopening flushed file at 1731752797989 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7898c099: reopening flushed file at 1731752797992 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@728e5d2: reopening flushed file at 1731752797995 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36d08e49: reopening flushed file at 1731752797998 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1700ms, sequenceid=90, compaction requested=false at 1731752798002 (+4 ms)Writing region close event to WAL at 1731752798003 (+1 ms)Running coprocessor post-close hooks at 1731752798006 (+3 ms)Closed at 1731752798006 2024-11-16T10:26:38,006 DEBUG [RS_CLOSE_META-regionserver/b16ff38458c1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T10:26:38,104 INFO [RS:0;b16ff38458c1:35873 {}] regionserver.HRegionServer(976): stopping server b16ff38458c1,35873,1731752627506; all regions closed. 2024-11-16T10:26:38,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741834_1010 (size=25344) 2024-11-16T10:26:38,111 DEBUG [RS:0;b16ff38458c1:35873 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/oldWALs 2024-11-16T10:26:38,111 INFO [RS:0;b16ff38458c1:35873 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b16ff38458c1%2C35873%2C1731752627506.meta:.meta(num 1731752631222) 2024-11-16T10:26:38,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741833_1009 (size=14050951) 2024-11-16T10:26:38,115 DEBUG [RS:0;b16ff38458c1:35873 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/oldWALs 2024-11-16T10:26:38,115 INFO [RS:0;b16ff38458c1:35873 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b16ff38458c1%2C35873%2C1731752627506:(num 1731752630673) 2024-11-16T10:26:38,115 DEBUG [RS:0;b16ff38458c1:35873 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T10:26:38,115 INFO [RS:0;b16ff38458c1:35873 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T10:26:38,115 INFO [RS:0;b16ff38458c1:35873 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T10:26:38,115 INFO [RS:0;b16ff38458c1:35873 {}] hbase.ChoreService(370): Chore service for: regionserver/b16ff38458c1:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-16T10:26:38,116 INFO [RS:0;b16ff38458c1:35873 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T10:26:38,116 INFO [regionserver/b16ff38458c1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T10:26:38,116 INFO [RS:0;b16ff38458c1:35873 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35873 2024-11-16T10:26:38,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T10:26:38,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35873-0x101432bffa20001, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b16ff38458c1,35873,1731752627506 2024-11-16T10:26:38,172 INFO [RS:0;b16ff38458c1:35873 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T10:26:38,183 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b16ff38458c1,35873,1731752627506] 2024-11-16T10:26:38,193 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b16ff38458c1,35873,1731752627506 already deleted, retry=false 2024-11-16T10:26:38,193 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b16ff38458c1,35873,1731752627506 expired; onlineServers=0 2024-11-16T10:26:38,193 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'b16ff38458c1,38809,1731752626047' ***** 2024-11-16T10:26:38,193 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-16T10:26:38,193 INFO [M:0;b16ff38458c1:38809 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T10:26:38,193 INFO [M:0;b16ff38458c1:38809 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T10:26:38,193 DEBUG [M:0;b16ff38458c1:38809 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T10:26:38,194 DEBUG [M:0;b16ff38458c1:38809 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T10:26:38,194 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T10:26:38,194 DEBUG [master/b16ff38458c1:0:becomeActiveMaster-HFileCleaner.large.0-1731752630314 {}] cleaner.HFileCleaner(306): Exit Thread[master/b16ff38458c1:0:becomeActiveMaster-HFileCleaner.large.0-1731752630314,5,FailOnTimeoutGroup] 2024-11-16T10:26:38,194 INFO [M:0;b16ff38458c1:38809 {}] hbase.ChoreService(370): Chore service for: master/b16ff38458c1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T10:26:38,194 INFO [M:0;b16ff38458c1:38809 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T10:26:38,194 DEBUG [master/b16ff38458c1:0:becomeActiveMaster-HFileCleaner.small.0-1731752630315 {}] cleaner.HFileCleaner(306): Exit Thread[master/b16ff38458c1:0:becomeActiveMaster-HFileCleaner.small.0-1731752630315,5,FailOnTimeoutGroup] 2024-11-16T10:26:38,194 DEBUG [M:0;b16ff38458c1:38809 {}] master.HMaster(1795): Stopping service threads 2024-11-16T10:26:38,194 INFO [M:0;b16ff38458c1:38809 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T10:26:38,194 INFO [M:0;b16ff38458c1:38809 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T10:26:38,194 ERROR [M:0;b16ff38458c1:38809 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:41487 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:41487,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-16T10:26:38,195 INFO [M:0;b16ff38458c1:38809 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T10:26:38,195 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T10:26:38,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T10:26:38,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T10:26:38,204 DEBUG [M:0;b16ff38458c1:38809 {}] zookeeper.ZKUtil(347): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-16T10:26:38,204 WARN [M:0;b16ff38458c1:38809 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-16T10:26:38,204 INFO [M:0;b16ff38458c1:38809 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/.lastflushedseqids 2024-11-16T10:26:38,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742428_1604 (size=72) 2024-11-16T10:26:38,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35873-0x101432bffa20001, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T10:26:38,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35873-0x101432bffa20001, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T10:26:38,283 INFO [RS:0;b16ff38458c1:35873 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T10:26:38,284 INFO [RS:0;b16ff38458c1:35873 {}] regionserver.HRegionServer(1031): Exiting; stopping=b16ff38458c1,35873,1731752627506; zookeeper connection closed. 2024-11-16T10:26:38,284 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@18a6d3cd {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@18a6d3cd 2024-11-16T10:26:38,285 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-16T10:26:38,613 INFO [M:0;b16ff38458c1:38809 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T10:26:38,613 INFO [M:0;b16ff38458c1:38809 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-16T10:26:38,614 DEBUG [M:0;b16ff38458c1:38809 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T10:26:38,614 INFO [M:0;b16ff38458c1:38809 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T10:26:38,614 DEBUG [M:0;b16ff38458c1:38809 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T10:26:38,614 DEBUG [M:0;b16ff38458c1:38809 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T10:26:38,614 DEBUG [M:0;b16ff38458c1:38809 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T10:26:38,614 INFO [M:0;b16ff38458c1:38809 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=768.44 KB heapSize=945.52 KB 2024-11-16T10:26:38,628 DEBUG [M:0;b16ff38458c1:38809 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/47d7806b3ade4dc281905b08b59fa27e is 82, key is hbase:meta,,1/info:regioninfo/1731752631298/Put/seqid=0 2024-11-16T10:26:38,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742429_1605 (size=5672) 2024-11-16T10:26:39,031 INFO [M:0;b16ff38458c1:38809 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2193 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/47d7806b3ade4dc281905b08b59fa27e 2024-11-16T10:26:39,057 DEBUG [M:0;b16ff38458c1:38809 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/606987f712c44182ba13f9e6db2d2ff5 is 2276, key is \x00\x00\x00\x00\x00\x00\x00\x8F/proc:d/1731752771792/Put/seqid=0 2024-11-16T10:26:39,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742430_1606 (size=43500) 2024-11-16T10:26:39,461 INFO [M:0;b16ff38458c1:38809 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=767.88 KB at sequenceid=2193 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/606987f712c44182ba13f9e6db2d2ff5 2024-11-16T10:26:39,465 INFO [M:0;b16ff38458c1:38809 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 606987f712c44182ba13f9e6db2d2ff5 2024-11-16T10:26:39,483 DEBUG [M:0;b16ff38458c1:38809 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bdbcd3aac6bb4fe5bfa288e14d1eeed0 is 69, key is b16ff38458c1,35873,1731752627506/rs:state/1731752630415/Put/seqid=0 2024-11-16T10:26:39,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742431_1607 (size=5156) 2024-11-16T10:26:39,887 INFO [M:0;b16ff38458c1:38809 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2193 (bloomFilter=true), to=hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bdbcd3aac6bb4fe5bfa288e14d1eeed0 2024-11-16T10:26:39,891 DEBUG [M:0;b16ff38458c1:38809 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/47d7806b3ade4dc281905b08b59fa27e as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/47d7806b3ade4dc281905b08b59fa27e 2024-11-16T10:26:39,894 INFO [M:0;b16ff38458c1:38809 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/47d7806b3ade4dc281905b08b59fa27e, entries=8, sequenceid=2193, filesize=5.5 K 2024-11-16T10:26:39,895 DEBUG [M:0;b16ff38458c1:38809 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/606987f712c44182ba13f9e6db2d2ff5 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/606987f712c44182ba13f9e6db2d2ff5 2024-11-16T10:26:39,898 INFO [M:0;b16ff38458c1:38809 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 606987f712c44182ba13f9e6db2d2ff5 2024-11-16T10:26:39,898 INFO [M:0;b16ff38458c1:38809 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/606987f712c44182ba13f9e6db2d2ff5, entries=166, sequenceid=2193, filesize=42.5 K 2024-11-16T10:26:39,898 DEBUG [M:0;b16ff38458c1:38809 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bdbcd3aac6bb4fe5bfa288e14d1eeed0 as hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/bdbcd3aac6bb4fe5bfa288e14d1eeed0 2024-11-16T10:26:39,901 INFO [M:0;b16ff38458c1:38809 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41487/user/jenkins/test-data/92946c95-d4bf-9c2a-55bd-a700a21ddfb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/bdbcd3aac6bb4fe5bfa288e14d1eeed0, entries=1, sequenceid=2193, filesize=5.0 K 2024-11-16T10:26:39,902 INFO [M:0;b16ff38458c1:38809 {}] regionserver.HRegion(3140): Finished flush of dataSize ~768.44 KB/786880, heapSize ~945.22 KB/967904, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1288ms, sequenceid=2193, compaction requested=false 2024-11-16T10:26:39,903 INFO [M:0;b16ff38458c1:38809 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T10:26:39,903 DEBUG [M:0;b16ff38458c1:38809 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731752798614Disabling compacts and flushes for region at 1731752798614Disabling writes for close at 1731752798614Obtaining lock to block concurrent updates at 1731752798614Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731752798614Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=786880, getHeapSize=968144, getOffHeapSize=0, getCellsCount=2384 at 1731752798614Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731752798615 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731752798615Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731752798627 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731752798628 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731752799034 (+406 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731752799057 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731752799057Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731752799465 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731752799482 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731752799483 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b26f899: reopening flushed file at 1731752799890 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@294bc14b: reopening flushed file at 1731752799894 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4773725c: reopening flushed file at 1731752799898 (+4 ms)Finished flush of dataSize ~768.44 KB/786880, heapSize ~945.22 KB/967904, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1288ms, sequenceid=2193, compaction requested=false at 1731752799902 (+4 ms)Writing region close event to WAL at 1731752799903 (+1 ms)Closed at 1731752799903 2024-11-16T10:26:39,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741830_1006 (size=930608) 2024-11-16T10:26:39,906 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T10:26:39,906 INFO [M:0;b16ff38458c1:38809 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T10:26:39,906 INFO [M:0;b16ff38458c1:38809 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38809 2024-11-16T10:26:39,906 INFO [M:0;b16ff38458c1:38809 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T10:26:40,061 INFO [M:0;b16ff38458c1:38809 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T10:26:40,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T10:26:40,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38809-0x101432bffa20000, quorum=127.0.0.1:49279, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T10:26:40,067 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T10:26:40,069 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T10:26:40,069 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T10:26:40,070 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T10:26:40,070 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/hadoop.log.dir/,STOPPED} 2024-11-16T10:26:40,074 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-16T10:26:40,074 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-16T10:26:40,074 WARN [BP-1320990832-172.17.0.2-1731752621574 heartbeating to localhost/127.0.0.1:41487 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-16T10:26:40,074 WARN [BP-1320990832-172.17.0.2-1731752621574 heartbeating to localhost/127.0.0.1:41487 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1320990832-172.17.0.2-1731752621574 (Datanode Uuid db4eb425-94e6-4ce5-acf1-af055905740e) service to localhost/127.0.0.1:41487 2024-11-16T10:26:40,076 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/cluster_7ec35abe-b1d1-d9da-1787-50b9d114d89d/data/data1/current/BP-1320990832-172.17.0.2-1731752621574 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T10:26:40,076 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/cluster_7ec35abe-b1d1-d9da-1787-50b9d114d89d/data/data2/current/BP-1320990832-172.17.0.2-1731752621574 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T10:26:40,078 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-16T10:26:40,087 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T10:26:40,087 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T10:26:40,088 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T10:26:40,088 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T10:26:40,088 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/09f67035-f0e4-9efc-59c5-da210c414c11/hadoop.log.dir/,STOPPED} 2024-11-16T10:26:40,106 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-16T10:26:40,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down